From 0f42b49c3da8e858ecd2cd8dec5c5b06752e6483 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt <92181998+umang8223@users.noreply.github.com> Date: Tue, 6 Jun 2023 21:56:38 -0400 Subject: [PATCH 001/213] Oss dockehubpush (#339) * Parametrize Docker image registry * Refactoring scripts for generalization --- docker/packager/binary/Dockerfile | 5 ++- docker/test/base/Dockerfile | 3 +- docker/test/codebrowser/Dockerfile | 3 +- docker/test/fasttest/Dockerfile | 3 +- docker/test/fuzzer/Dockerfile | 3 +- docker/test/integration/base/Dockerfile | 3 +- docker/test/keeper-jepsen/Dockerfile | 3 +- docker/test/server-jepsen/Dockerfile | 3 +- docker/test/sqllogic/Dockerfile | 3 +- docker/test/stateful/Dockerfile | 3 +- docker/test/stateless/Dockerfile | 3 +- docker/test/stress/Dockerfile | 3 +- docker/test/unit/Dockerfile | 3 +- docker/test/upgrade/Dockerfile | 3 +- tests/ci/clickhouse_helper.py | 3 +- tests/ci/docker_images_check.py | 13 ++++--- tests/ci/docker_manifests_merge.py | 4 +- tests/ci/env_helper.py | 8 ++++ tests/ci/get_robot_token.py | 49 ++++++++++++++++++------- tests/ci/s3_helper.py | 3 +- tests/ci/worker/prepare-ci-ami.sh | 2 +- 21 files changed, 86 insertions(+), 40 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index dd21c8552d3b..ae8309953905 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/binary-builder . ARG FROM_TAG=latest -FROM clickhouse/test-util:latest AS cctools +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:latest AS cctools # The cctools are built always from the clickhouse/test-util:latest and cached inline # Theoretically, it should improve rebuild speed significantly ENV CC=clang-${LLVM_VERSION} @@ -39,7 +40,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ # END COMPILE # !!!!!!!!!!! -FROM clickhouse/test-util:$FROM_TAG +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index f68368044541..1ad6f7ab8c07 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/test-base . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 8136fd1fbbcd..e6676c740fd3 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -2,7 +2,8 @@ # docker build --network=host -t clickhouse/codebrowser . # docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser ARG FROM_TAG=latest -FROM clickhouse/binary-builder:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/binary-builder:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index ffb13fc774d4..98d58fa9767c 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fasttest . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index aa71074c02a7..7401edc4eefe 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index de8efa20af4e..6f442be44666 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/integration-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG SHELL ["/bin/bash", "-c"] diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index a794e076ec02..b0425847048d 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/keeper-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index a212427b2a1a..45c28bc6b036 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/server-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 83dcf7e1f56a..f30e3e0b86ed 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/sqllogic-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 71a2e92e3a86..d69662997915 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #47031 # docker build -t clickhouse/stateful-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 40109255a7e5..42d7b1053a3a 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stateless-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index e9712f430fd2..7e4e17da2a68 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stress-test . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index b75bfb6661cc..d389e0d3f341 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/unit-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get install gdb diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile index 8e5890b81a0c..86055ff312a9 100644 --- a/docker/test/upgrade/Dockerfile +++ b/docker/test/upgrade/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/upgrade-check . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 64b64896f661..c30d0900cc16 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -7,6 +7,7 @@ import requests # type: ignore from get_robot_token import get_parameter_from_ssm +from env_helper import GITHUB_REPOSITORY from pr_info import PRInfo from report import TestResults @@ -141,7 +142,7 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" + pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/master" base_ref = "master" head_ref = "master" base_repo = pr_info.repo_full_name diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 16a58a90dcf6..a128f41ab040 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -15,7 +15,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -94,7 +94,7 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( - pr_info: PRInfo, images_dict: ImagesDict + pr_info: PRInfo, images_dict: ImagesDict, DOCKER_REPO ) -> Set[DockerImage]: if not images_dict: return set() @@ -113,7 +113,7 @@ def get_changed_docker_images( for dockerfile_dir, image_description in images_dict.items(): for f in files_changed: if f.startswith(dockerfile_dir): - name = image_description["name"] + name = DOCKER_REPO + "/" + image_description["name"] only_amd64 = image_description.get("only_amd64", False) logging.info( "Found changed file '%s' which affects " @@ -137,7 +137,7 @@ def get_changed_docker_images( dependent, image, ) - name = images_dict[dependent]["name"] + name = DOCKER_REPO + "/" + images_dict[dependent]["name"] only_amd64 = images_dict[dependent].get("only_amd64", False) changed_images.append(DockerImage(dependent, name, only_amd64, image)) index += 1 @@ -248,6 +248,7 @@ def build_and_push_one_image( "docker buildx build --builder default " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" + f"--build-arg DOCKER_REPO={DOCKER_REPO} " # A hack to invalidate cache, grep for it in docker/ dir f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " @@ -400,7 +401,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, @@ -424,7 +425,7 @@ def main(): # If the event does not contain diff, nothing will be built pass - changed_images = get_changed_docker_images(pr_info, images_dict) + changed_images = get_changed_docker_images(pr_info, images_dict, DOCKER_REPO) if changed_images: logging.info( "Has changed images: %s", ", ".join([im.path for im in changed_images]) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index d89708b9277e..8c740db61844 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -11,7 +11,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import RUNNER_TEMP +from env_helper import RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -174,7 +174,7 @@ def main(): args = parse_args() if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 9303f9ae2936..f8a7e4ba727c 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -14,6 +14,8 @@ CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB", "local") +DOCKER_REPO = os.getenv("DOCKER_REPO","docker.io") +DOCKER_USER = os.getenv("DOCKER_USER", "robotclickhouse") GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") @@ -23,8 +25,14 @@ REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports"))) REPO_COPY = os.getenv("REPO_COPY", git_root) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) +S3_REGION = os.getenv("S3_REGION", "us-east-1") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") +S3_ENDPOINT = os.getenv("S3_ENDPOINT","https://s3.amazonaws.com") +VAULT_PATH = os.getenv("VAULT_PATH") +VAULT_TOKEN = os.getenv("VAULT_TOKEN") +VAULT_URL = os.getenv("VAULT_URL") +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b41eba49cc32..ba3086e90ae7 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -4,9 +4,11 @@ from typing import Optional import boto3 # type: ignore +import hvac from github import Github from github.AuthenticatedUser import AuthenticatedUser +from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT @dataclass class Token: @@ -16,9 +18,15 @@ class Token: def get_parameter_from_ssm(name, decrypt=True, client=None): - if not client: - client = boto3.client("ssm", region_name="us-east-1") - return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + if VAULT_URL: + if not client: + client = hvac.Client(url=VAULT_URL,token=VAULT_TOKEN) + parameter = client.secrets.kv.v2.read_secret_version(mount_point=VAULT_MOUNT_POINT,path=VAULT_PATH)["data"]["data"][name] + else: + if not client: + client = boto3.client("ssm", region_name="us-east-1") + parameter = client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + return parameter ROBOT_TOKEN = None # type: Optional[Token] @@ -28,18 +36,31 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value - client = boto3.client("ssm", region_name="us-east-1") - parameters = client.describe_parameters( - ParameterFilters=[ - {"Key": "Name", "Option": "BeginsWith", "Values": [token_prefix_env_name]} - ] - )["Parameters"] - assert parameters - - for token_name in [p["Name"] for p in parameters]: - value = get_parameter_from_ssm(token_name, True, client) + + client = None + values = [] + + if VAULT_URL: + client = hvac.Client(url=VAULT_URL,token=VAULT_TOKEN) + parameters = client.secrets.kv.v2.read_secret_version(mount_point=VAULT_MOUNT_POINT,path=VAULT_PATH)["data"]["data"] + parameters = {key: value for key, value in parameters.items() if key.startswith(token_prefix_env_name)} + assert parameters + values = list(parameters.values()) + else: + client = boto3.client("ssm", region_name="us-east-1") + parameters = client.describe_parameters( + ParameterFilters=[ + {"Key": "Name", "Option": "BeginsWith", "Values": [token_prefix_env_name]} + ] + )["Parameters"] + assert parameters + for token_name in [p["Name"] for p in parameters]: + value = get_parameter_from_ssm(token_name, True, client) + values.append(value) + + for value in values: gh = Github(value, per_page=100) - # Do not spend additional request to API by accessin user.login unless + # Do not spend additional request to API by accessing user.login unless # the token is chosen by the remaining requests number user = gh.get_user() rest, _ = gh.rate_limiting diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 9ee0a4312947..441ebf006f08 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -15,6 +15,7 @@ from env_helper import ( S3_TEST_REPORTS_BUCKET, S3_BUILDS_BUCKET, + S3_REGION, RUNNER_TEMP, CI, S3_URL, @@ -47,7 +48,7 @@ class S3Helper: def __init__(self): config = botocore.config.Config(max_pool_connections=self.max_pool_size) - self.session = boto3.session.Session(region_name="us-east-1") + self.session = boto3.session.Session(region_name=S3_REGION) self.client = self.session.client("s3", endpoint_url=S3_URL, config=config) self.host = S3_URL self.download_host = S3_DOWNLOAD diff --git a/tests/ci/worker/prepare-ci-ami.sh b/tests/ci/worker/prepare-ci-ami.sh index 20e7e3fd53e2..59de58d884b1 100644 --- a/tests/ci/worker/prepare-ci-ami.sh +++ b/tests/ci/worker/prepare-ci-ami.sh @@ -90,7 +90,7 @@ systemctl restart docker sudo -u ubuntu docker buildx version sudo -u ubuntu docker buildx create --use --name default-builder -pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory +pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory hvac mkdir -p $RUNNER_HOME && cd $RUNNER_HOME From eb6b820a543e63282980da7541a0650b9c03e94e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt <92181998+umang8223@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:33:24 -0400 Subject: [PATCH 002/213] Updated for rebuild of style-test --- .github/workflows/master.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f0741b5465f3..57c099121219 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -9,19 +9,8 @@ on: # yamllint disable-line rule:truthy branches: - 'master' jobs: - PythonUnitTests: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Python unit tests - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 -m unittest discover -s . -p '*_test.py' DockerHubPushAarch64: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -53,7 +42,7 @@ jobs: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] + needs: [DockerHubPushAmd64, DockerHubPushAarch64] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code From dc5eeedb13b2a3094abc79e6effb9d3e641086d0 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 6 Jun 2023 22:08:26 -0400 Subject: [PATCH 003/213] testing for checklabels --- .github/workflows/pull_request.yml | 32 +++++++++++++++--------------- tests/ci/run_check.py | 18 ++++++++--------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index afc08f3e6376..1bfa6d13242f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -36,22 +36,22 @@ jobs: run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 run_check.py - PythonUnitTests: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Python unit tests - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - echo "Testing the main ci directory" - python3 -m unittest discover -s . -p '*_test.py' - for dir in *_lambda/; do - echo "Testing $dir" - python3 -m unittest discover -s "$dir" -p '*_test.py' - done + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done DockerHubPushAarch64: needs: CheckLabels runs-on: [self-hosted, style-checker-aarch64] diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 330a1309016c..f36513fd81c1 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -5,6 +5,15 @@ from github import Github +from docs_check import NAME as DOCS_NAME +from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL +from get_robot_token import get_best_robot_token +from pr_info import FORCE_TESTS_LABEL, PRInfo +from lambda_shared_package.lambda_shared.pr import ( + CATEGORY_TO_LABEL, + TRUSTED_CONTRIBUTORS, + check_pr_description, +) from commit_status_helper import ( CI_STATUS_NAME, NotSet, @@ -16,15 +25,6 @@ remove_labels, set_mergeable_check, ) -from docs_check import NAME as DOCS_NAME -from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL -from get_robot_token import get_best_robot_token -from pr_info import FORCE_TESTS_LABEL, PRInfo -from lambda_shared_package.lambda_shared.pr import ( - CATEGORY_TO_LABEL, - TRUSTED_CONTRIBUTORS, - check_pr_description, -) TRUSTED_ORG_IDS = { 54801242, # clickhouse From ec2680cc7dcf45d641115b9876be639d595b3699 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 6 Jun 2023 22:53:27 -0400 Subject: [PATCH 004/213] testing for checklabels and commented code --- .github/workflows/pull_request.yml | 9402 ++++++++++++++-------------- 1 file changed, 4701 insertions(+), 4701 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1bfa6d13242f..ca8df8c764db 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -178,68 +178,68 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - CompatibilityCheckX86: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckX86 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - CompatibilityCheckAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckAarch64 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # CompatibilityCheckX86: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/compatibility_check + # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + # REPORTS_PATH=${{runner.temp}}/reports_dir + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: CompatibilityCheckX86 + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # CompatibilityCheckAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/compatibility_check + # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + # REPORTS_PATH=${{runner.temp}}/reports_dir + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: CompatibilityCheckAarch64 + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" ######################################################################################### #################################### ORDINARY BUILDS #################################### ######################################################################################### @@ -285,4642 +285,4642 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" - BuilderBinRelease: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_asan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebUBsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_ubsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebTsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_tsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebMsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_msan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebDebug: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_debug - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Apply sparse checkout for contrib # in order to check that it doesn't break build - run: | - rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' - git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' - "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' - du -hs "$GITHUB_WORKSPACE/contrib" ||: - find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -########################################################################################## -##################################### SPECIAL BUILDS ##################################### -########################################################################################## - BuilderBinClangTidy: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_tidy - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwin: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinFreeBSD: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_freebsd - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinPPC64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_ppc64le - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAmd64Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_amd64_compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64V80Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64_v80compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -############################################################################################ -##################################### Docker images ####################################### -############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -##################################### BUILD REPORTER ####################################### -############################################################################################ - BuilderReport: - needs: - - BuilderBinRelease - - BuilderDebAarch64 - - BuilderDebAsan - - BuilderDebDebug - - BuilderDebMsan - - BuilderDebRelease - - BuilderDebTsan - - BuilderDebUBsan - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=ClickHouse build check - REPORTS_PATH=${{runner.temp}}/reports_dir - TEMP_PATH=${{runner.temp}}/report_check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderSpecialReport: - needs: - - BuilderBinAarch64 - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - - BuilderBinFreeBSD - - BuilderBinPPC64 - - BuilderBinAmd64Compat - - BuilderBinAarch64V80Compat - - BuilderBinClangTidy - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/report_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=ClickHouse special build check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -#################################### INSTALL PACKAGES ###################################### -############################################################################################ - InstallPackagesTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (amd64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -########################### FUNCTIONAl STATELESS TESTS ####################################### -############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseWideParts: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_wide_parts - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, wide parts enabled) - REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseAnalyzer: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_analyzer - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, analyzer) - REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug5: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan0: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan1: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan3: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan4: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan5: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_flaky_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests flaky check (asan) - REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - TestsBugfixCheck: - needs: [CheckLabels, StyleCheck] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/tests_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=tests bugfix validate check - KILL_TIMEOUT=3600 - REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Bugfix test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - - TEMP_PATH="${TEMP_PATH}/integration" \ - REPORTS_PATH="${REPORTS_PATH}/integration" \ - python3 integration_test_check.py "Integration $CHECK_NAME" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - TEMP_PATH="${TEMP_PATH}/stateless" \ - REPORTS_PATH="${REPORTS_PATH}/stateless" \ - python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -############################ FUNCTIONAl STATEFUL TESTS ####################################### -############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - # Parallel replicas - FunctionalStatefulTestDebugParallelReplicas: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsanParallelReplicas: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsanParallelReplicas: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsanParallelReplicas: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsanParallelReplicas: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestReleaseParallelReplicas: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -######################################### STRESS TESTS ####################################### -############################################################################################## - StressTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (asan) - REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestTsan: - needs: [BuilderDebTsan] - # func testers have 16 cores + 128 GB memory - # while stress testers have 36 cores + 72 memory - # It would be better to have something like 32 + 128, - # but such servers almost unavailable as spot instances. - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (tsan) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (msan) - REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_undefined - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (ubsan) - REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (debug) - REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################## - ######################################### UPGRADE CHECK ###################################### - ############################################################################################## - UpgradeCheckAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (asan) - REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckTsan: - needs: [BuilderDebTsan] - # same as for stress test with tsan - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (tsan) - REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (msan) - REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (debug) - REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -##################################### AST FUZZERS ############################################ -############################################################################################## - ASTFuzzerTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (asan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (tsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestUBSan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (ubsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestMSan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (msan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (debug) - REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -############################# INTEGRATION TESTS ############################################# -############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan4: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan5: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan5: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests flaky check (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### UNIT TESTS ############################################# -############################################################################################# - UnitTestsAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (asan) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsReleaseClang: - needs: [BuilderBinRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (release-clang) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (tsan) - REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (msan) - REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (ubsan) - REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### PERFORMANCE TESTS ###################################### -############################################################################################# - PerformanceComparisonX86-0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-0: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-1: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-2: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-3: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -###################################### SQLANCER FUZZERS ###################################### -############################################################################################## - SQLancerTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (release) - REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - SQLancerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (debug) - REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -###################################### JEPSEN TESTS ######################################### -############################################################################################# - Jepsen: - # This is special test NOT INCLUDED in FinishCheck - # When it's skipped, all dependent tasks will be skipped too. - # DO NOT add it there - if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') - needs: [BuilderBinRelease] - uses: ./.github/workflows/jepsen.yml - FinishCheck: - needs: - - StyleCheck - - DockerHubPush - - DockerServerImages - - CheckLabels - - BuilderReport - - BuilderSpecialReport - - FastTest - - FunctionalStatelessTestDebug0 - - FunctionalStatelessTestDebug1 - - FunctionalStatelessTestDebug2 - - FunctionalStatelessTestDebug3 - - FunctionalStatelessTestDebug4 - - FunctionalStatelessTestRelease - - FunctionalStatelessTestReleaseDatabaseReplicated0 - - FunctionalStatelessTestReleaseDatabaseReplicated1 - - FunctionalStatelessTestReleaseDatabaseReplicated2 - - FunctionalStatelessTestReleaseDatabaseReplicated3 - - FunctionalStatelessTestReleaseWideParts - - FunctionalStatelessTestReleaseAnalyzer - - FunctionalStatelessTestAarch64 - - FunctionalStatelessTestAsan0 - - FunctionalStatelessTestAsan1 - - FunctionalStatelessTestAsan2 - - FunctionalStatelessTestAsan3 - - FunctionalStatelessTestTsan0 - - FunctionalStatelessTestTsan1 - - FunctionalStatelessTestTsan2 - - FunctionalStatelessTestTsan3 - - FunctionalStatelessTestTsan4 - - FunctionalStatelessTestMsan0 - - FunctionalStatelessTestMsan1 - - FunctionalStatelessTestMsan2 - - FunctionalStatelessTestMsan3 - - FunctionalStatelessTestMsan4 - - FunctionalStatelessTestMsan5 - - FunctionalStatelessTestUBsan0 - - FunctionalStatelessTestUBsan1 - - FunctionalStatefulTestDebug - - FunctionalStatefulTestRelease - - FunctionalStatefulTestAarch64 - - FunctionalStatefulTestAsan - - FunctionalStatefulTestTsan - - FunctionalStatefulTestMsan - - FunctionalStatefulTestUBsan - - FunctionalStatelessTestReleaseS3_0 - - FunctionalStatelessTestReleaseS3_1 - - FunctionalStatelessTestS3Debug0 - - FunctionalStatelessTestS3Debug1 - - FunctionalStatelessTestS3Debug2 - - FunctionalStatelessTestS3Debug4 - - FunctionalStatelessTestS3Debug5 - - FunctionalStatelessTestS3Tsan0 - - FunctionalStatelessTestS3Tsan1 - - FunctionalStatelessTestS3Tsan2 - - FunctionalStatelessTestS3Tsan4 - - StressTestDebug - - StressTestAsan - - StressTestTsan - - StressTestMsan - - StressTestUBsan - - ASTFuzzerTestDebug - - ASTFuzzerTestAsan - - ASTFuzzerTestTsan - - ASTFuzzerTestMSan - - ASTFuzzerTestUBSan - - IntegrationTestsAsan0 - - IntegrationTestsAsan1 - - IntegrationTestsAsan2 - - IntegrationTestsAsan3 - - IntegrationTestsAsan4 - - IntegrationTestsAsan5 - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 - - IntegrationTestsRelease2 - - IntegrationTestsRelease3 - - IntegrationTestsTsan0 - - IntegrationTestsTsan1 - - IntegrationTestsTsan2 - - IntegrationTestsTsan3 - - IntegrationTestsTsan4 - - IntegrationTestsTsan5 - - PerformanceComparisonX86-0 - - PerformanceComparisonX86-1 - - PerformanceComparisonX86-2 - - PerformanceComparisonX86-3 - - PerformanceComparisonAarch-0 - - PerformanceComparisonAarch-1 - - PerformanceComparisonAarch-2 - - PerformanceComparisonAarch-3 - - UnitTestsAsan - - UnitTestsTsan - - UnitTestsMsan - - UnitTestsUBsan - - UnitTestsReleaseClang - - CompatibilityCheckX86 - - CompatibilityCheckAarch64 - - IntegrationTestsFlakyCheck - - SQLancerTestRelease - - SQLancerTestDebug - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Finish label - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 finish_check.py - python3 merge_pr.py --check-approved -############################################################################################## -########################### SQLLOGIC TEST ################################################### -############################################################################################## - SQLLogicTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqllogic_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Sqllogic test (release) - REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Sqllogic test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/images_path +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# fetch-depth: 0 # for performance artifact +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_asan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebUBsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_ubsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebTsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_tsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebMsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_msan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebDebug: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_debug +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ +# DockerServerImages: +# needs: +# - BuilderDebRelease +# - BuilderDebAarch64 +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself +# - name: Check docker clickhouse/clickhouse-server building +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-server --image-path docker/server +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: +# needs: +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# InstallPackagesTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## +# FunctionalStatelessTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan0: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan1: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan1: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## +# FunctionalStatefulTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## +# StressTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (asan) +# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestTsan: +# needs: [BuilderDebTsan] +# # func testers have 16 cores + 128 GB memory +# # while stress testers have 36 cores + 72 memory +# # It would be better to have something like 32 + 128, +# # but such servers almost unavailable as spot instances. +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (tsan) +# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (msan) +# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_undefined +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (ubsan) +# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (debug) +# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# +# IntegrationTestsAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" From c5820741706d015d61a114989c35eea09be373da Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 6 Jun 2023 23:03:15 -0400 Subject: [PATCH 005/213] updated needs for job --- .github/workflows/pull_request.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ca8df8c764db..92659dba770a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -54,7 +54,7 @@ jobs: # done DockerHubPushAarch64: needs: CheckLabels - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -87,7 +87,7 @@ jobs: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] + needs: [DockerHubPushAmd64, DockerHubPushAarch64] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -150,7 +150,7 @@ jobs: sudo rm -fr "$TEMP_PATH" FastTest: needs: DockerHubPush - runs-on: [self-hosted, builder] + runs-on: [self-hosted, style-checker] steps: - name: Set envs run: | @@ -245,7 +245,7 @@ jobs: ######################################################################################### BuilderDebRelease: needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] + runs-on: [self-hosted, style-checker] steps: - name: Set envs run: | From 928b125c9548cc0108e1d7a0e462d0e4e51ba929 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 6 Jun 2023 23:55:40 -0400 Subject: [PATCH 006/213] updated style_check.py --- tests/ci/style_check.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 33a5cd21f39b..0b09d24d886a 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -22,8 +22,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP -from get_robot_token import get_best_robot_token +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo @@ -152,6 +152,13 @@ def main(): if args.push: checkout_head(pr_info) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + gh = GitHub(get_best_robot_token(), create_cache_dir=False) commit = get_commit(gh, pr_info.sha) @@ -169,7 +176,7 @@ def main(): if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(temp_path, "clickhouse/style-test") + docker_image = get_image_with_version(temp_path, f"{DOCKER_REPO}/clickhouse/style-test") s3_helper = S3Helper() cmd = ( From eb912531436364a319ac9f320a6b201e2023ca44 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 00:25:23 -0400 Subject: [PATCH 007/213] minor changes to style-test Dockerfile --- docker/test/style/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 746cc7bb2d5b..454e241fc6c9 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -10,6 +10,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ aspell \ curl \ + # curl \ git \ file \ libxml2-utils \ From 2aea26438c7a65adc7f7a4d3d41b2e708b3a6501 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 11:16:25 -0400 Subject: [PATCH 008/213] removing curl --- docker/test/style/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 454e241fc6c9..82db59c9799e 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -9,8 +9,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ aspell \ - curl \ - # curl \ + curl \ git \ file \ libxml2-utils \ From d83d10ce98d2fe1fbaeb0e50bc540ab4b096d73e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 11:17:51 -0400 Subject: [PATCH 009/213] adding push --- .github/workflows/pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 92659dba770a..63741dc892aa 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,6 +5,7 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy + push: pull_request: types: - synchronize From 547acbb35d57196b6a527deb4a9fc4c3ac0ecc89 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 7 Jun 2023 08:32:56 -0700 Subject: [PATCH 010/213] Automatic style fix --- tests/ci/docker_images_check.py | 8 +++++++- tests/ci/env_helper.py | 4 ++-- tests/ci/get_robot_token.py | 29 ++++++++++++++++++++++------- tests/ci/style_check.py | 4 +++- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index a128f41ab040..9a5ef39437a5 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -15,7 +15,13 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL, DOCKER_USER, DOCKER_REPO +from env_helper import ( + GITHUB_WORKSPACE, + RUNNER_TEMP, + GITHUB_RUN_URL, + DOCKER_USER, + DOCKER_REPO, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index f8a7e4ba727c..ce2a31260318 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -14,7 +14,7 @@ CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB", "local") -DOCKER_REPO = os.getenv("DOCKER_REPO","docker.io") +DOCKER_REPO = os.getenv("DOCKER_REPO", "docker.io") DOCKER_USER = os.getenv("DOCKER_USER", "robotclickhouse") GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") @@ -28,7 +28,7 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") -S3_ENDPOINT = os.getenv("S3_ENDPOINT","https://s3.amazonaws.com") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.amazonaws.com") VAULT_PATH = os.getenv("VAULT_PATH") VAULT_TOKEN = os.getenv("VAULT_TOKEN") VAULT_URL = os.getenv("VAULT_URL") diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index ba3086e90ae7..1e7580211426 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -10,6 +10,7 @@ from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT + @dataclass class Token: user: AuthenticatedUser @@ -20,12 +21,16 @@ class Token: def get_parameter_from_ssm(name, decrypt=True, client=None): if VAULT_URL: if not client: - client = hvac.Client(url=VAULT_URL,token=VAULT_TOKEN) - parameter = client.secrets.kv.v2.read_secret_version(mount_point=VAULT_MOUNT_POINT,path=VAULT_PATH)["data"]["data"][name] + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameter = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"][name] else: if not client: client = boto3.client("ssm", region_name="us-east-1") - parameter = client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + parameter = client.get_parameter(Name=name, WithDecryption=decrypt)[ + "Parameter" + ]["Value"] return parameter @@ -41,16 +46,26 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): values = [] if VAULT_URL: - client = hvac.Client(url=VAULT_URL,token=VAULT_TOKEN) - parameters = client.secrets.kv.v2.read_secret_version(mount_point=VAULT_MOUNT_POINT,path=VAULT_PATH)["data"]["data"] - parameters = {key: value for key, value in parameters.items() if key.startswith(token_prefix_env_name)} + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameters = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"] + parameters = { + key: value + for key, value in parameters.items() + if key.startswith(token_prefix_env_name) + } assert parameters values = list(parameters.values()) else: client = boto3.client("ssm", region_name="us-east-1") parameters = client.describe_parameters( ParameterFilters=[ - {"Key": "Name", "Option": "BeginsWith", "Values": [token_prefix_env_name]} + { + "Key": "Name", + "Option": "BeginsWith", + "Values": [token_prefix_env_name], + } ] )["Parameters"] assert parameters diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 0b09d24d886a..87a770b3e022 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -176,7 +176,9 @@ def main(): if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(temp_path, f"{DOCKER_REPO}/clickhouse/style-test") + docker_image = get_image_with_version( + temp_path, f"{DOCKER_REPO}/clickhouse/style-test" + ) s3_helper = S3Helper() cmd = ( From 461528969c670ebfebc974a20578ef0eefa3a442 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 11:44:00 -0400 Subject: [PATCH 011/213] removed extra space --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 82db59c9799e..746cc7bb2d5b 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -9,7 +9,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ aspell \ - curl \ + curl \ git \ file \ libxml2-utils \ From 4217ab5bdb584e20cf2ed652014e181767fee363 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 12:01:53 -0400 Subject: [PATCH 012/213] removed to run on push --- .github/workflows/pull_request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 63741dc892aa..92659dba770a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5,7 +5,6 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy - push: pull_request: types: - synchronize From f05beaf1f838088a29b573675beed8e3ca175880 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 12:36:13 -0400 Subject: [PATCH 013/213] updated docker_test.yaml --- tests/ci/docker_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 9d68f4364391..aa69785425ea 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -5,7 +5,7 @@ from unittest.mock import patch, MagicMock from pathlib import Path -from env_helper import GITHUB_RUN_URL +from env_helper import GITHUB_RUN_URL, DOCKER_REPO from pr_info import PRInfo from report import TestResult import docker_images_check as di @@ -32,7 +32,7 @@ def test_get_changed_docker_images(self): images = sorted( list( di.get_changed_docker_images( - pr_info, di.get_images_dict("/", self.docker_images_path) + pr_info, di.get_images_dict("/", self.docker_images_path), DOCKER_REPO ) ) ) @@ -129,6 +129,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version --cache-from type=registry,ref=name:version " "--cache-from type=registry,ref=name:latest " @@ -146,6 +147,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version2 " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " @@ -162,6 +164,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " @@ -180,6 +183,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " From db5e4b698ed15fe493052797ec081a93336a9e65 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 12:40:37 -0400 Subject: [PATCH 014/213] testing for commit --- .github/workflows/pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 92659dba770a..6884a6e8bb4a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,6 +4,7 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + on: # yamllint disable-line rule:truthy pull_request: types: From 7a99f069fb4d8cb0031a290e5ad78b64d130256c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 13:33:00 -0400 Subject: [PATCH 015/213] testing for run --- tests/ci/get_robot_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 1e7580211426..b8b3e5634721 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -2,7 +2,7 @@ import logging from dataclasses import dataclass from typing import Optional - +# import boto3 # type: ignore import hvac from github import Github From f5bc35dde8d7c82d8e5cfb66ce1b42e2c9ab6f0e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 13:37:02 -0400 Subject: [PATCH 016/213] removing commented line --- tests/ci/get_robot_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b8b3e5634721..1e7580211426 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -2,7 +2,7 @@ import logging from dataclasses import dataclass from typing import Optional -# + import boto3 # type: ignore import hvac from github import Github From a952f275909243b9153ff9a86e59d45570dfafd8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 15:19:14 -0400 Subject: [PATCH 017/213] updated to python3 --- tests/ci/version_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/version_test.py b/tests/ci/version_test.py index abd0f9349f4b..50ba8017f471 100644 --- a/tests/ci/version_test.py +++ b/tests/ci/version_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import unittest from argparse import ArgumentTypeError From 6f19f9b5c2759c6676887be7c69014543f59a268 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 17:22:43 -0400 Subject: [PATCH 018/213] added annotation --- tests/ci/docker_images_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 9a5ef39437a5..25ede4394f8c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -100,7 +100,7 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( - pr_info: PRInfo, images_dict: ImagesDict, DOCKER_REPO + pr_info: PRInfo, images_dict: ImagesDict, DOCKER_REPO: str ) -> Set[DockerImage]: if not images_dict: return set() From bfba6457e57dfc2b660bb8a75f5d1b8d94032b5c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 17:43:53 -0400 Subject: [PATCH 019/213] updated fastest to download icr image --- tests/ci/fast_test_check.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 89066ade2cb3..b448ea46b510 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -23,8 +23,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH -from get_robot_token import get_best_robot_token +from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -106,6 +106,13 @@ def main(): pr_info = PRInfo() + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) @@ -119,8 +126,9 @@ def main(): sys.exit(1) sys.exit(0) - docker_image = get_image_with_version(temp_path, "clickhouse/fasttest") - + docker_image = get_image_with_version( + temp_path, f"{DOCKER_REPO}/clickhouse/fasttest" + ) s3_helper = S3Helper() workspace = os.path.join(temp_path, "fasttest-workspace") From fac251f3eab3574f41a48d08f9ba2ffcbbb68b77 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 18:21:22 -0400 Subject: [PATCH 020/213] testing to remove circular dependency --- .github/workflows/pull_request.yml | 3 --- tests/ci/build_download_helper.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6884a6e8bb4a..11abdc5afd38 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -241,9 +241,6 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -######################################################################################### -#################################### ORDINARY BUILDS #################################### -######################################################################################### BuilderDebRelease: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, style-checker] diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 47c11ee0911d..64035feaaa81 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -8,9 +8,9 @@ from pathlib import Path from typing import Any, Callable, List +import get_robot_token as grt # we need an updated ROBOT_TOKEN import requests # type: ignore -import get_robot_token as grt # we need an updated ROBOT_TOKEN from ci_config import CI_CONFIG DOWNLOAD_RETRIES_COUNT = 5 From 8b6b14ebd754eede276bb63ad891be096bdb8c75 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 21:33:25 -0400 Subject: [PATCH 021/213] swithed import statement --- tests/ci/style_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 87a770b3e022..835643fda238 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -22,8 +22,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo From 2af6c2ebd683ceaf99fc4124d5846426dcb88448 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 7 Jun 2023 23:35:55 -0400 Subject: [PATCH 022/213] testing for cyclic import --- tests/ci/get_robot_token.py | 2 +- tests/ci/style_check.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 1e7580211426..b990f3e42989 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -8,7 +8,7 @@ from github import Github from github.AuthenticatedUser import AuthenticatedUser -from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT +from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT, GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO @dataclass diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 835643fda238..5f189bfc3406 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -22,8 +22,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm, GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO +# from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo From fafb27f3def2af231e78a245ce7975d46f12f60c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 00:10:57 -0400 Subject: [PATCH 023/213] testing for cyclic import --- tests/ci/build_download_helper.py | 4 +++- tests/ci/get_robot_token.py | 2 +- tests/ci/style_check.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 64035feaaa81..b3af504cccd5 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -8,9 +8,9 @@ from pathlib import Path from typing import Any, Callable, List -import get_robot_token as grt # we need an updated ROBOT_TOKEN import requests # type: ignore +# import get_robot_token as grt # we need an updated ROBOT_TOKEN from ci_config import CI_CONFIG DOWNLOAD_RETRIES_COUNT = 5 @@ -53,6 +53,8 @@ def get_gh_api( It sets auth automatically when ROBOT_TOKEN is already set by get_best_robot_token """ + import get_robot_token as grt + def set_auth_header(): if "headers" in kwargs: if "Authorization" not in kwargs["headers"]: diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b990f3e42989..1e7580211426 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -8,7 +8,7 @@ from github import Github from github.AuthenticatedUser import AuthenticatedUser -from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT, GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO +from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT @dataclass diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 5f189bfc3406..835643fda238 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -22,8 +22,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from get_robot_token import get_best_robot_token, get_parameter_from_ssm, GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO -# from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo From d4f2889d70c3081ef6a8c7da636dfc0ea1839da7 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 00:12:41 -0400 Subject: [PATCH 024/213] commneted style and fast test --- .github/workflows/pull_request.yml | 133 +++++++++++++++-------------- 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 11abdc5afd38..912c1ace7484 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -114,71 +114,71 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, style-checker] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{ runner.temp }}/style_check - ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # StyleCheck: + # needs: DockerHubPush + # runs-on: [self-hosted, style-checker] + # # We need additional `&& ! cancelled()` to have the job being able to cancel + # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{ runner.temp }}/style_check + # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test + # run: | + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # CompatibilityCheckX86: # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] @@ -242,7 +242,8 @@ jobs: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" BuilderDebRelease: - needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] runs-on: [self-hosted, style-checker] steps: - name: Set envs From 0093ce91018a094884bb25718439bb7b5616f77d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 00:39:49 -0400 Subject: [PATCH 025/213] updated build_check.yaml --- tests/ci/build_check.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 4bc61c79fc03..ca841d6a2f87 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -23,8 +23,10 @@ S3_BUILDS_BUCKET, S3_DOWNLOAD, TEMP_PATH, + DOCKER_USER, + DOCKER_REPO, ) -from get_robot_token import get_best_robot_token +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub from pr_info import PRInfo from s3_helper import S3Helper @@ -36,7 +38,7 @@ update_version_local, ) -IMAGE_NAME = "clickhouse/binary-builder" +IMAGE_NAME = f"{DOCKER_REPO}/clickhouse/binary-builder" BUILD_LOG_NAME = "build_log.log" @@ -82,6 +84,9 @@ def get_packager_cmd( cmd += f" --docker-image-version={image_version}" cmd += f" --version={build_version}" + cmd += f" --docker-repo={DOCKER_REPO}" + cmd += f" --docker-user={DOCKER_USER}" + cmd += " --docker-password={}".format(get_parameter_from_ssm("dockerhub_robot_password")) if _can_export_binaries(build_config): cmd += " --with-binaries=tests" @@ -294,6 +299,13 @@ def main(): # put them as github actions artifact (result) check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + "docker login {} --username '{}' --password-stdin".format(DOCKER_REPO, DOCKER_USER), + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + # If it's a latter running, we need to mark possible failed status mark_failed_reports_pending(build_name, pr_info) From 8123402984eb4a2e277e0a906036edeb89b3269f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 00:50:36 -0400 Subject: [PATCH 026/213] updated packager file --- docker/packager/packager | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 1b3df858cd2e..9959bf0b26a5 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -10,7 +10,7 @@ from typing import List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary" -IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}-builder" +# IMAGE_NAME = f"{args.docker_repo}/clickhouse/{IMAGE_TYPE}-builder" class BuildException(Exception): @@ -423,6 +423,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--as-root", action="store_true", help="if the container should run as root" ) + parser.add_argument( + "--docker-repo", default="docker.io", help="docker repository" + ) + parser.add_argument( + "--docker-user", default=argparse.SUPPRESS, help="docker user" + ) + parser.add_argument( + "--docker-password", default=argparse.SUPPRESS, help="docker password" + ) args = parser.parse_args() @@ -454,6 +463,8 @@ def main(): ch_root = args.clickhouse_repo_path + IMAGE_NAME = f"{args.docker_repo}/clickhouse/{IMAGE_TYPE}-builder" + dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile" image_with_version = IMAGE_NAME + ":" + args.docker_image_version if args.force_build_image: @@ -479,7 +490,13 @@ def main(): args.with_coverage, args.with_binaries, ) - + if args.docker_user: + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + "docker login {} --username '{}' --password-stdin".format(args.docker_repo, args.docker_user), + input=args.docker_password, + encoding="utf-8", + shell=True, + ) pre_build(args.clickhouse_repo_path, env_prepared) run_docker_image_with_env( image_with_version, From 9b3ce1c74aba9e735a5ddb69a9652cccfc846be9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 09:59:44 -0400 Subject: [PATCH 027/213] added hvac in image --- .github/workflows/pull_request.yml | 208 ++++++++++++++--------------- docker/test/style/Dockerfile | 2 +- tests/ci/build_check.py | 2 +- tests/ci/docker_server.py | 6 +- 4 files changed, 109 insertions(+), 109 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 912c1ace7484..ed46c768ac3a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -114,73 +114,104 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - # StyleCheck: - # needs: DockerHubPush - # runs-on: [self-hosted, style-checker] - # # We need additional `&& ! cancelled()` to have the job being able to cancel - # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{ runner.temp }}/style_check - # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{ runner.temp }}/style_check + ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/fasttest + REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} + - name: Fast Test + run: | + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # CompatibilityCheckX86: + # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/fasttest - # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches + # TEMP_PATH=${{runner.temp}}/compatibility_check + # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + # REPORTS_PATH=${{runner.temp}}/reports_dir # EOF # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true - # - name: Download changed images + # - name: Download json reports # uses: actions/download-artifact@v3 # with: - # name: changed_images - # path: ${{ env.TEMP_PATH }} - # - name: Fast Test + # path: ${{ env.REPORTS_PATH }} + # - name: CompatibilityCheckX86 # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # CompatibilityCheckX86: - # needs: [BuilderDebRelease] + # sudo rm -fr "$TEMP_PATH" + # CompatibilityCheckAarch64: + # needs: [BuilderDebAarch64] # runs-on: [self-hosted, style-checker] # steps: # - name: Set envs @@ -198,92 +229,61 @@ jobs: # uses: actions/download-artifact@v3 # with: # path: ${{ env.REPORTS_PATH }} - # - name: CompatibilityCheckX86 + # - name: CompatibilityCheckAarch64 # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions + # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - # CompatibilityCheckAarch64: - # needs: [BuilderDebAarch64] + # BuilderDebRelease: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, style-checker] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/compatibility_check - # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - # REPORTS_PATH=${{runner.temp}}/reports_dir + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: CompatibilityCheckAarch64 + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - BuilderDebRelease: - # needs: [DockerHubPush, FastTest, StyleCheck] - needs: [DockerHubPush] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 746cc7bb2d5b..b27df72795df 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ + && pip3 install black==23.1.0 boto3 hvac codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ && apt-get clean \ && rm -rf /root/.cache/pip diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index ca841d6a2f87..2ba723aa88d7 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -300,7 +300,7 @@ def main(): check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config) subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login {} --username '{}' --password-stdin".format(DOCKER_REPO, DOCKER_USER), + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index a434d3cc8411..5230e110c9e3 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -17,7 +17,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage -from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD +from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm from git_helper import Git from pr_info import PRInfo @@ -337,12 +337,12 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, ) - NAME = f"Docker image {image.repo} build and push" + NAME = f"Docker image {image.repo} build and push (actions)" logging.info("Following tags will be created: %s", ", ".join(tags)) status = "success" From f0bf2e300a9f4253d3818df3ad45f96a94ce8214 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 10:33:54 -0400 Subject: [PATCH 028/213] added hvac in image --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index b27df72795df..1e075b3f6ffc 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==23.1.0 boto3 hvac codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ + && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 hvac \ && apt-get clean \ && rm -rf /root/.cache/pip From 77253f5bb52d2c7411f56a9b1c82849aa530b984 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:03:15 -0400 Subject: [PATCH 029/213] added hvac in image --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 1e075b3f6ffc..b27df72795df 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 hvac \ + && pip3 install black==23.1.0 boto3 hvac codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ && apt-get clean \ && rm -rf /root/.cache/pip From 894801e3113e31886910208cdea70732801c7487 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:11:10 -0400 Subject: [PATCH 030/213] commented check labels --- .github/workflows/pull_request.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ed46c768ac3a..f7ed7b7d182b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -24,19 +24,19 @@ on: # yamllint disable-line rule:truthy ##################################### SMALL CHECKS ####################################### ########################################################################################## jobs: - CheckLabels: - runs-on: [self-hosted, style-checker] - # Run the first check always, even if the CI is cancelled - if: ${{ always() }} - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py # PythonUnitTests: # runs-on: [self-hosted, style-checker] # steps: From d63d644ee39a2886f87ccbe3f179b406309b7a93 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:12:22 -0400 Subject: [PATCH 031/213] removed needs --- .github/workflows/pull_request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f7ed7b7d182b..ada01568e9c0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -54,7 +54,7 @@ jobs: # python3 -m unittest discover -s "$dir" -p '*_test.py' # done DockerHubPushAarch64: - needs: CheckLabels + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -71,7 +71,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - needs: CheckLabels + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code From 9847f87277c9f26919f17c1ac706bb75696baffa Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:33:30 -0400 Subject: [PATCH 032/213] undo hvac installation --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index b27df72795df..746cc7bb2d5b 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install black==23.1.0 boto3 hvac codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ + && pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \ && apt-get clean \ && rm -rf /root/.cache/pip From 4e01265c0a0fb3ecdfacd5aa2d8b034a255c0b8b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:40:36 -0400 Subject: [PATCH 033/213] commneted not required jobs --- .github/workflows/pull_request.yml | 245 +++++++++++++++-------------- 1 file changed, 123 insertions(+), 122 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ada01568e9c0..ea21da73e865 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,7 +4,6 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - on: # yamllint disable-line rule:truthy pull_request: types: @@ -24,19 +23,19 @@ on: # yamllint disable-line rule:truthy ##################################### SMALL CHECKS ####################################### ########################################################################################## jobs: - # CheckLabels: - # runs-on: [self-hosted, style-checker] - # # Run the first check always, even if the CI is cancelled - # if: ${{ always() }} - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Labels check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 run_check.py + CheckLabels: + runs-on: [self-hosted, style-checker] + # Run the first check always, even if the CI is cancelled + if: ${{ always() }} + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Labels check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 run_check.py # PythonUnitTests: # runs-on: [self-hosted, style-checker] # steps: @@ -52,9 +51,9 @@ jobs: # for dir in *_lambda/; do # echo "Testing $dir" # python3 -m unittest discover -s "$dir" -p '*_test.py' - # done + # done DockerHubPushAarch64: - # needs: CheckLabels + needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -71,7 +70,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - # needs: CheckLabels + needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -179,111 +178,113 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # CompatibilityCheckX86: - # needs: [BuilderDebRelease] - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/compatibility_check - # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - # REPORTS_PATH=${{runner.temp}}/reports_dir - # EOF - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: CompatibilityCheckX86 - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" - # CompatibilityCheckAarch64: - # needs: [BuilderDebAarch64] - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/compatibility_check - # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - # REPORTS_PATH=${{runner.temp}}/reports_dir - # EOF - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: CompatibilityCheckAarch64 - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" - # BuilderDebRelease: - # # needs: [DockerHubPush, FastTest, StyleCheck] - # needs: [DockerHubPush, FastTest, StyleCheck] - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_release - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # fetch-depth: 0 # for performance artifact - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### +# BuilderDebRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# fetch-depth: 0 # for performance artifact +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] From d2c3e5a4915b21f966387b7527873bd52d186244 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 11:54:28 -0400 Subject: [PATCH 034/213] undo import changes --- tests/ci/build_download_helper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index b3af504cccd5..8bdcb61eedef 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -10,7 +10,7 @@ import requests # type: ignore -# import get_robot_token as grt # we need an updated ROBOT_TOKEN +import get_robot_token as grt # we need an updated ROBOT_TOKEN from ci_config import CI_CONFIG DOWNLOAD_RETRIES_COUNT = 5 @@ -52,8 +52,6 @@ def get_gh_api( "403 rate limit exceeded" error It sets auth automatically when ROBOT_TOKEN is already set by get_best_robot_token """ - - import get_robot_token as grt def set_auth_header(): if "headers" in kwargs: From e7448691244684502b8bfc0fd092321e50b7694e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 12:41:04 -0400 Subject: [PATCH 035/213] added type ignore --- tests/ci/get_robot_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 1e7580211426..b43be7f533a0 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -4,7 +4,7 @@ from typing import Optional import boto3 # type: ignore -import hvac +import hvac # type: ignore from github import Github from github.AuthenticatedUser import AuthenticatedUser From 4c240d37af6fbffd3d447a94635fbc7c82b6e5ea Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 13:11:27 -0400 Subject: [PATCH 036/213] corrected indentation --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ea21da73e865..9194810b9274 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -51,7 +51,7 @@ jobs: # for dir in *_lambda/; do # echo "Testing $dir" # python3 -m unittest discover -s "$dir" -p '*_test.py' - # done + # done DockerHubPushAarch64: needs: CheckLabels runs-on: [self-hosted, style-checker] From 6649a8d3848d4eb38e7377879f2e3341a763d3e0 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 15:30:23 -0400 Subject: [PATCH 037/213] testing --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9194810b9274..a6282c7d6114 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -178,7 +178,7 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# CompatibilityCheckX86: +## CompatibilityCheckX86: # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] # steps: From da27737750afb4fae4b9b3375edf59d376255af8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 15:56:59 -0400 Subject: [PATCH 038/213] style fixing --- docker/packager/packager | 14 ++++---------- tests/ci/build_check.py | 4 ++-- tests/ci/build_download_helper.py | 2 +- tests/ci/docker_server.py | 10 +++++++++- tests/ci/docker_test.py | 4 +++- tests/ci/get_robot_token.py | 2 +- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 9959bf0b26a5..ee248bef6308 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -423,15 +423,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--as-root", action="store_true", help="if the container should run as root" ) - parser.add_argument( - "--docker-repo", default="docker.io", help="docker repository" - ) - parser.add_argument( - "--docker-user", default=argparse.SUPPRESS, help="docker user" - ) - parser.add_argument( - "--docker-password", default=argparse.SUPPRESS, help="docker password" - ) + parser.add_argument("--docker-repo", default="docker.io", help="docker repository") + parser.add_argument("--docker-user", default=argparse.SUPPRESS, help="docker user") + parser.add_argument("--docker-password", default=argparse.SUPPRESS, help="docker password") args = parser.parse_args() @@ -492,7 +486,7 @@ def main(): ) if args.docker_user: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login {} --username '{}' --password-stdin".format(args.docker_repo, args.docker_user), + f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", input=args.docker_password, encoding="utf-8", shell=True, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 2ba723aa88d7..fc9cc4170cf6 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -24,7 +24,7 @@ S3_DOWNLOAD, TEMP_PATH, DOCKER_USER, - DOCKER_REPO, + DOCKER_REPO, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -86,7 +86,7 @@ def get_packager_cmd( cmd += f" --version={build_version}" cmd += f" --docker-repo={DOCKER_REPO}" cmd += f" --docker-user={DOCKER_USER}" - cmd += " --docker-password={}".format(get_parameter_from_ssm("dockerhub_robot_password")) + cmd += f" --docker-password={get_parameter_from_ssm('dockerhub_robot_password')}" if _can_export_binaries(build_config): cmd += " --with-binaries=tests" diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 8bdcb61eedef..47c11ee0911d 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -52,7 +52,7 @@ def get_gh_api( "403 rate limit exceeded" error It sets auth automatically when ROBOT_TOKEN is already set by get_best_robot_token """ - + def set_auth_header(): if "headers" in kwargs: if "Authorization" not in kwargs["headers"]: diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 5230e110c9e3..d3612c873c18 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -17,7 +17,15 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage -from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD, DOCKER_USER, DOCKER_REPO +from env_helper import ( + CI, + GITHUB_RUN_URL, + RUNNER_TEMP, + S3_BUILDS_BUCKET, + S3_DOWNLOAD, + DOCKER_USER, + DOCKER_REPO, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from git_helper import Git from pr_info import PRInfo diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index aa69785425ea..8ccfa471871c 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -32,7 +32,9 @@ def test_get_changed_docker_images(self): images = sorted( list( di.get_changed_docker_images( - pr_info, di.get_images_dict("/", self.docker_images_path), DOCKER_REPO + pr_info, + di.get_images_dict("/", self.docker_images_path), + DOCKER_REPO, ) ) ) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b43be7f533a0..8f49f29fa971 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -4,7 +4,7 @@ from typing import Optional import boto3 # type: ignore -import hvac # type: ignore +import hvac # type: ignore from github import Github from github.AuthenticatedUser import AuthenticatedUser From c8f416a6afe86acb124102304f5ae43e38cf4325 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 16:22:10 -0400 Subject: [PATCH 039/213] parameterizing github repo in fasttest dockerfile --- .github/workflows/pull_request.yml | 30 +++++++++++++++--------------- docker/test/fasttest/run.sh | 2 +- tests/ci/docker_images_check.py | 2 ++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a6282c7d6114..c75343b7ca0d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,19 +23,19 @@ on: # yamllint disable-line rule:truthy ##################################### SMALL CHECKS ####################################### ########################################################################################## jobs: - CheckLabels: - runs-on: [self-hosted, style-checker] - # Run the first check always, even if the CI is cancelled - if: ${{ always() }} - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py # PythonUnitTests: # runs-on: [self-hosted, style-checker] # steps: @@ -53,7 +53,7 @@ jobs: # python3 -m unittest discover -s "$dir" -p '*_test.py' # done DockerHubPushAarch64: - needs: CheckLabels + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -70,7 +70,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - needs: CheckLabels + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index dab873377ce5..7573858059c8 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -81,7 +81,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/${GITHUB_REPOSITORY}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 25ede4394f8c..110dbc139489 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -21,6 +21,7 @@ GITHUB_RUN_URL, DOCKER_USER, DOCKER_REPO, + GITHUB_REPOSITORY, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo @@ -255,6 +256,7 @@ def build_and_push_one_image( f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" f"--build-arg DOCKER_REPO={DOCKER_REPO} " + f"--build-arg GITHUB_REPOSITORY={GITHUB_REPOSITORY} " # A hack to invalidate cache, grep for it in docker/ dir f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " From 4649c915b33b87ba07ea3b1713aa724e17ad0f82 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 16:32:32 -0400 Subject: [PATCH 040/213] coorection of variable --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 7573858059c8..5d6ce88fd764 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -81,7 +81,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/${GITHUB_REPOSITORY}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/$GITHUB_REPOSITORY.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From 040904cfffdbad4aa167e45277f7922435269ddb Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 16:42:24 -0400 Subject: [PATCH 041/213] export varible --- docker/test/fasttest/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 5d6ce88fd764..5d7e212e56d6 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -37,6 +37,7 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH +export GITHUB_REPOSITORY function ccache_status { @@ -81,7 +82,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/$GITHUB_REPOSITORY.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY".git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From 32624117ba81b543837d2700abb52b7c9384a1b1 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 16:50:13 -0400 Subject: [PATCH 042/213] export varible --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 5d7e212e56d6..aaacff67b602 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -82,7 +82,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY".git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/$GITHUB_REPOSITORY.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From b549178f7d349467ca9760ed8f8ec72ebfde9323 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:10:23 -0400 Subject: [PATCH 043/213] export varible --- docker/test/fasttest/Dockerfile | 2 ++ docker/test/fasttest/run.sh | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 98d58fa9767c..4b7ca4709407 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -4,6 +4,8 @@ ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG +ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY + RUN apt-get update \ && apt-get install \ brotli \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index aaacff67b602..7573858059c8 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -37,7 +37,6 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH -export GITHUB_REPOSITORY function ccache_status { @@ -82,7 +81,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/$GITHUB_REPOSITORY.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/${GITHUB_REPOSITORY}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From 247504d5cf4bb98e0d68e67ce1ad70036c107e65 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:18:36 -0400 Subject: [PATCH 044/213] export varible --- docker/test/fasttest/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 7573858059c8..25e96752a153 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -37,6 +37,7 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH +export GITHUB_REPOSITORY function ccache_status { From f7d78bde39589ae9e8be9d2468b1e3d2fa297de5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:23:31 -0400 Subject: [PATCH 045/213] echo export varible --- docker/test/fasttest/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 25e96752a153..631feab69eb2 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -38,6 +38,7 @@ export FASTTEST_DATA export FASTTEST_OUT export PATH export GITHUB_REPOSITORY +echo "GITHUB_REPOSITORY: $GITHUB_REPOSITORY" function ccache_status { From 04151b7a4bc5ab9480abf64d26dd2026a30884a9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:29:24 -0400 Subject: [PATCH 046/213] echo export varible --- tests/ci/docker_images_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 110dbc139489..1c8d635d669c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -265,6 +265,7 @@ def build_and_push_one_image( f"{push_arg}" f"--progress plain {image.full_path}" ) + print("GITHUB_REPOSITORY:", GITHUB_REPOSITORY) logging.info("Docker command to run: %s", cmd) with TeePopen(cmd, build_log) as proc: retcode = proc.wait() From 62e2691c37c45f03fefa953a60313f3153926f9a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:53:42 -0400 Subject: [PATCH 047/213] echo export varible --- docker/test/fasttest/Dockerfile | 3 ++- docker/test/fasttest/run.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 4b7ca4709407..fb9eef7675cb 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -42,4 +42,5 @@ ENV PULL_REQUEST_NUMBER='' ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 COPY run.sh / -CMD ["/bin/bash", "/run.sh"] +CMD ["/bin/bash", "/run.sh", "$GITHUB_REPOSITORY"] +# CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 631feab69eb2..161d239a50fd 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -29,6 +29,8 @@ FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}") FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}") PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" +GITHUB_REPOSITORY=$1 + # Export these variables, so that all subsequent invocations of the script # use them, and not try to guess them anew, which leads to weird effects. export FASTTEST_WORKSPACE @@ -37,7 +39,6 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH -export GITHUB_REPOSITORY echo "GITHUB_REPOSITORY: $GITHUB_REPOSITORY" function ccache_status From 307404626961f4cb478c32a95fb6dcc319fb4326 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 17:59:41 -0400 Subject: [PATCH 048/213] echo export varible --- docker/test/fasttest/run.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 161d239a50fd..b4ac6eac9247 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -29,8 +29,6 @@ FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}") FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}") PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" -GITHUB_REPOSITORY=$1 - # Export these variables, so that all subsequent invocations of the script # use them, and not try to guess them anew, which leads to weird effects. export FASTTEST_WORKSPACE @@ -39,7 +37,11 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH -echo "GITHUB_REPOSITORY: $GITHUB_REPOSITORY" +export GITHUB_REPOSITORY + +echo "GITHUB_REPOSITORY: $1" +GITHUB_REPOSITORY=$1 + function ccache_status { From cc295960a719a4f9bdbcca11f74b630611187120 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 18:18:50 -0400 Subject: [PATCH 049/213] hardcoded repo and style fix --- docker/packager/packager | 4 +++- docker/test/fasttest/run.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index ee248bef6308..30c41ee3b504 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -425,7 +425,9 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--docker-repo", default="docker.io", help="docker repository") parser.add_argument("--docker-user", default=argparse.SUPPRESS, help="docker user") - parser.add_argument("--docker-password", default=argparse.SUPPRESS, help="docker password") + parser.add_argument( + "--docker-password", default=argparse.SUPPRESS, help="docker password" + ) args = parser.parse_args() diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index b4ac6eac9247..2134b16becae 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -86,7 +86,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/${GITHUB_REPOSITORY}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/ClibMouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From e5c983d07d241ac941575f7ba498f557e8ead667 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 8 Jun 2023 18:27:54 -0400 Subject: [PATCH 050/213] hardcoded repo and fixed some errors --- docker/test/fasttest/Dockerfile | 3 +-- docker/test/fasttest/run.sh | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index fb9eef7675cb..4b7ca4709407 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -42,5 +42,4 @@ ENV PULL_REQUEST_NUMBER='' ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 COPY run.sh / -CMD ["/bin/bash", "/run.sh", "$GITHUB_REPOSITORY"] -# CMD ["/bin/bash", "/run.sh"] +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 2134b16becae..0735b03bfa26 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -37,11 +37,6 @@ export FASTTEST_BUILD export FASTTEST_DATA export FASTTEST_OUT export PATH -export GITHUB_REPOSITORY - -echo "GITHUB_REPOSITORY: $1" -GITHUB_REPOSITORY=$1 - function ccache_status { From 11ba7295a613fdc9ff58ad9697a55dbefbd12c7f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 12:41:50 -0400 Subject: [PATCH 051/213] testing for run --- .github/workflows/pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c75343b7ca0d..0f5dcc9ca92e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -53,6 +53,7 @@ jobs: # python3 -m unittest discover -s "$dir" -p '*_test.py' # done DockerHubPushAarch64: + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: From e3ca350ddcdc51584968cd9919ac38227a0ad968 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 13:53:29 -0400 Subject: [PATCH 052/213] troubleshooting --- docker/test/fasttest/run.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 0735b03bfa26..1b19b787366a 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -61,6 +61,9 @@ function start_server clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" & set +m + # Add a log message to indicate the server startup process has begun + echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup initiated" >> "$FASTTEST_OUTPUT/server.log" + for _ in {1..60}; do if clickhouse-client --query "select 1"; then break @@ -76,6 +79,9 @@ function start_server local server_pid server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")" echo "ClickHouse server pid '$server_pid' started and responded" + + # Add a log message to indicate the server startup process has completed + echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup completed" >> "$FASTTEST_OUTPUT/server.log" } function clone_root From 44b007fddd4a2ff919a46fecb31d32b116083075 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 15:58:14 -0400 Subject: [PATCH 053/213] test for run --- .github/workflows/pull_request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0f5dcc9ca92e..c75343b7ca0d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -53,7 +53,6 @@ jobs: # python3 -m unittest discover -s "$dir" -p '*_test.py' # done DockerHubPushAarch64: - # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: From 81a55cbe588d84a12c0bff9eb412a531ce8f58c3 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 16:06:20 -0400 Subject: [PATCH 054/213] test for run --- .github/workflows/pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c75343b7ca0d..aed788e873a8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -53,6 +53,7 @@ jobs: # python3 -m unittest discover -s "$dir" -p '*_test.py' # done DockerHubPushAarch64: + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: From a87e160a8831149ae82a6117613c1f090cf3037e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 17:04:50 -0400 Subject: [PATCH 055/213] adding creds --- tests/ci/env_helper.py | 2 ++ tests/ci/fast_test_check.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index ce2a31260318..61820576c2b8 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,6 +10,8 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) +AWS_ACCESS_KEY_ID=("AWS_ACCESS_KEY_ID", "") +AWS_SECRET_ACCESS_KEY=("AWS_SECRET_ACCESS_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index b448ea46b510..171a62ce078e 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -23,7 +23,7 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO +from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results @@ -48,6 +48,9 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + f"-e AWS_DEFAULT_REGION={S3_REGION} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From 338cdd70d35699f938177abb34f116ab9934a829 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 17:11:44 -0400 Subject: [PATCH 056/213] correction --- tests/ci/env_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 61820576c2b8..a01d12e91ab9 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,8 +10,8 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) -AWS_ACCESS_KEY_ID=("AWS_ACCESS_KEY_ID", "") -AWS_SECRET_ACCESS_KEY=("AWS_SECRET_ACCESS_KEY", "") +AWS_ACCESS_KEY_ID=os.getenv("AWS_ACCESS_KEY_ID", "") +AWS_SECRET_ACCESS_KEY=os.getenv("AWS_SECRET_ACCESS_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") From e6eea1042dcbcfb32ba9e93e81a5328d5bc0825b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 9 Jun 2023 17:33:02 -0400 Subject: [PATCH 057/213] added endpoint --- tests/ci/fast_test_check.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 171a62ce078e..04587fc86a32 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -23,7 +23,7 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_ENDPOINT from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results @@ -50,7 +50,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " - f"-e AWS_DEFAULT_REGION={S3_REGION} " + f"-e AWS_DEFAULT_REGION={S3_REGION} " + f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From 1ca4eb7507e7c9a1811c58619adaf197275be1dd Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 10:06:39 -0400 Subject: [PATCH 058/213] commented region --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 04587fc86a32..246e15479559 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -50,7 +50,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " - f"-e AWS_DEFAULT_REGION={S3_REGION} " + # f"-e AWS_DEFAULT_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" From d7b20e21290c03f14cc858fe5a34379c13fb69d0 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 10:15:57 -0400 Subject: [PATCH 059/213] commented secret and access key_id --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 246e15479559..68cc5af59bb6 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -48,8 +48,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + # f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + # f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " # f"-e AWS_DEFAULT_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 2964b3bbf493f7c6d93e6d5f98535e25ce6974c5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 10:26:06 -0400 Subject: [PATCH 060/213] style fix --- tests/ci/fast_test_check.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 68cc5af59bb6..ff49903419f8 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -23,7 +23,16 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, DOCKER_USER, DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_ENDPOINT +from env_helper import ( + S3_BUILDS_BUCKET, + TEMP_PATH, + DOCKER_USER, + DOCKER_REPO, + S3_REGION, + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + S3_ENDPOINT, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results @@ -48,8 +57,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - # f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + # f"-e AWS_ACCESS_KEY_ID = {AWS_ACCESS_KEY_ID} " + # f"-e AWS_SECRET_ACCESS_KEY = {AWS_SECRET_ACCESS_KEY} " # f"-e AWS_DEFAULT_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 22a5cd4eb019d40fb9a8bed127ed35cc51b6a37b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 12:05:59 -0400 Subject: [PATCH 061/213] passing API key as --- tests/ci/env_helper.py | 4 ++-- tests/ci/fast_test_check.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a01d12e91ab9..5cb236ed446f 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,8 +10,8 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) -AWS_ACCESS_KEY_ID=os.getenv("AWS_ACCESS_KEY_ID", "") -AWS_SECRET_ACCESS_KEY=os.getenv("AWS_SECRET_ACCESS_KEY", "") +# AWS_ACCESS_KEY_ID=os.getenv("AWS_ACCESS_KEY_ID", "") +IBM_CLOUD_API_KEY=os.getenv("IBM_CLOUD_API_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ff49903419f8..5d97070bc431 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -30,7 +30,7 @@ DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, + IBM_CLOUD_API_KEY, S3_ENDPOINT, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -58,8 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID = {AWS_ACCESS_KEY_ID} " - # f"-e AWS_SECRET_ACCESS_KEY = {AWS_SECRET_ACCESS_KEY} " - # f"-e AWS_DEFAULT_REGION={S3_REGION} " + f"-e IBM_CLOUD_API_KEY = {IBM_CLOUD_API_KEY} " + f"-e IBM_CLOUD_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" From e3f7a62e66c3678b53561120ebb8be6b4a3a9bca Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 12:34:16 -0400 Subject: [PATCH 062/213] uncommented --- tests/ci/env_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 5cb236ed446f..0d679540f038 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,8 +10,8 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) -# AWS_ACCESS_KEY_ID=os.getenv("AWS_ACCESS_KEY_ID", "") -IBM_CLOUD_API_KEY=os.getenv("IBM_CLOUD_API_KEY", "") +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") +IBM_CLOUD_API_KEY = os.getenv("IBM_CLOUD_API_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") From 7e15b2d7718f8e2c53ddb86a6992b01c88287495 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 10 Jun 2023 12:46:29 -0400 Subject: [PATCH 063/213] removed space --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 5d97070bc431..859793652f4c 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,7 +58,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID = {AWS_ACCESS_KEY_ID} " - f"-e IBM_CLOUD_API_KEY = {IBM_CLOUD_API_KEY} " + f"-e IBM_CLOUD_API_KEY={IBM_CLOUD_API_KEY} " f"-e IBM_CLOUD_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 6494d04330f1bab9b7260e6338e04306b26b8548 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 11 Jun 2023 11:49:21 -0400 Subject: [PATCH 064/213] testing --- tests/ci/env_helper.py | 2 +- tests/ci/fast_test_check.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 0d679540f038..9802fe8e5a9c 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -11,7 +11,7 @@ TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") -IBM_CLOUD_API_KEY = os.getenv("IBM_CLOUD_API_KEY", "") +AWS_SECRET_ACCESS_KEY = os.getenv("IBM_CLOUD_API_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 859793652f4c..15775848b7c6 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -30,7 +30,7 @@ DOCKER_REPO, S3_REGION, AWS_ACCESS_KEY_ID, - IBM_CLOUD_API_KEY, + AWS_SECRET_ACCESS_KEY, S3_ENDPOINT, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -58,8 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID = {AWS_ACCESS_KEY_ID} " - f"-e IBM_CLOUD_API_KEY={IBM_CLOUD_API_KEY} " - f"-e IBM_CLOUD_REGION={S3_REGION} " + f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + f"-e AWS_DEFAULT_REGION={S3_REGION} " f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" From eab3c4d6af896364e42cf8da8e1692488ed87796 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 11 Jun 2023 22:20:24 -0400 Subject: [PATCH 065/213] added crn --- tests/ci/env_helper.py | 3 ++- tests/ci/fast_test_check.py | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 9802fe8e5a9c..1d72c13a5499 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,8 +10,9 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) +IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") -AWS_SECRET_ACCESS_KEY = os.getenv("IBM_CLOUD_API_KEY", "") +IBM_COS_API_KEY = os.getenv("IBM_CLOUD_API_KEY", "") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 15775848b7c6..1f2b37bd42c0 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -29,8 +29,8 @@ DOCKER_USER, DOCKER_REPO, S3_REGION, - AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, + IBM_COS_INSTANCE_CRN, + IBM_COS_API_KEY, S3_ENDPOINT, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -50,17 +50,16 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( f"docker run --cap-add=SYS_PTRACE " - "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID = {AWS_ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " - f"-e AWS_DEFAULT_REGION={S3_REGION} " - f"-e S3_ENDPOINT={S3_ENDPOINT} " + f"-e IBM_COS_API_KEY = {IBM_COS_API_KEY} " + f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " + f"-e IBM_COS_REGION={S3_REGION} " + # f"-e S3_ENDPOINT={S3_ENDPOINT} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From 7689e006c0c1e041f315cfb97ab9b16b90060041 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 11 Jun 2023 22:38:50 -0400 Subject: [PATCH 066/213] testing --- tests/ci/env_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 1d72c13a5499..710854c66d08 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,6 +10,7 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) + IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") IBM_COS_API_KEY = os.getenv("IBM_CLOUD_API_KEY", "") From 6cf3721c89eb9030f585d9cd6c260d07f31d9f55 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 11 Jun 2023 22:45:09 -0400 Subject: [PATCH 067/213] testing --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 1f2b37bd42c0..1e56141bf2de 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -56,7 +56,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e IBM_COS_API_KEY = {IBM_COS_API_KEY} " + f"-e IBM_COS_API_KEY={IBM_COS_API_KEY} " f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " f"-e IBM_COS_REGION={S3_REGION} " # f"-e S3_ENDPOINT={S3_ENDPOINT} " From ca224c38961d47f3aef9c724f224d8217e014917 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 08:54:28 -0400 Subject: [PATCH 068/213] checking for new change --- tests/ci/env_helper.py | 9 ++++----- tests/ci/fast_test_check.py | 11 ++++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 710854c66d08..47dd83ed6663 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -32,17 +32,16 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") -S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.amazonaws.com") -VAULT_PATH = os.getenv("VAULT_PATH") -VAULT_TOKEN = os.getenv("VAULT_TOKEN") -VAULT_URL = os.getenv("VAULT_URL") -VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) +VAULT_PATH = os.getenv("VAULT_PATH") +VAULT_TOKEN = os.getenv("VAULT_TOKEN") +VAULT_URL = os.getenv("VAULT_URL") +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 1e56141bf2de..c207e88de1d5 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -31,7 +31,7 @@ S3_REGION, IBM_COS_INSTANCE_CRN, IBM_COS_API_KEY, - S3_ENDPOINT, + S3_URL, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo @@ -55,11 +55,12 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e IBM_COS_API_KEY={IBM_COS_API_KEY} " - f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " - f"-e IBM_COS_REGION={S3_REGION} " - # f"-e S3_ENDPOINT={S3_ENDPOINT} " + # f"-e IBM_COS_API_KEY={IBM_COS_API_KEY} " + # f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " + f"-e SCCACHE_REGION={S3_REGION} " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From b341992f03dec6da23262cb94bcd3ccb44fac1dd Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 09:03:54 -0400 Subject: [PATCH 069/213] checking for new change --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index c207e88de1d5..82580cb121a9 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -50,6 +50,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( f"docker run --cap-add=SYS_PTRACE " + "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " From 35b9b1e432a2f4168fd85f1cbc61d8efeee67390 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 09:07:01 -0400 Subject: [PATCH 070/213] trying for SCCACHE_S3_NO_CREDENTIALS --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 82580cb121a9..ba524b465897 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,6 +58,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e SCCACHE_S3_NO_CREDENTIALS=true " # f"-e IBM_COS_API_KEY={IBM_COS_API_KEY} " # f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " f"-e SCCACHE_REGION={S3_REGION} " From 7a15e28c8e878ab9c6636e590bfbf2dd172188fc Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 09:38:56 -0400 Subject: [PATCH 071/213] removed SCCACHE_S3_NO_CREDENTIAL --- tests/ci/env_helper.py | 6 +++--- tests/ci/fast_test_check.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 47dd83ed6663..8aba88c14522 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -11,9 +11,9 @@ TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) -IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") -AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "") -IBM_COS_API_KEY = os.getenv("IBM_CLOUD_API_KEY", "") +IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "secret") +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "secret") +AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "secret") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ba524b465897..8915149078fe 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -29,8 +29,8 @@ DOCKER_USER, DOCKER_REPO, S3_REGION, - IBM_COS_INSTANCE_CRN, - IBM_COS_API_KEY, + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, S3_URL, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -59,8 +59,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e SCCACHE_S3_NO_CREDENTIALS=true " - # f"-e IBM_COS_API_KEY={IBM_COS_API_KEY} " - # f"-e IBM_COS_INSTANCE_CRN={IBM_COS_INSTANCE_CRN} " + f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 1eebf397ee8cd591f18c66e0cc3b348a37a5091a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 09:53:50 -0400 Subject: [PATCH 072/213] removed SCCACHE_S3_NO_CREDENTIAL --- tests/ci/fast_test_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 8915149078fe..bad73d68fae0 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,7 +58,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e SCCACHE_S3_NO_CREDENTIALS=true " f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " From 57742806cfadde08a7479d6108406c597994e821 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 11:02:04 -0400 Subject: [PATCH 073/213] testing after removing creds and see if it works --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index bad73d68fae0..3c84eef4ece4 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,8 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + # f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + # f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 2cc9ec46ad874c47cfcc477ad0e2f96560f392c5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 11:26:23 -0400 Subject: [PATCH 074/213] added timeout --- tests/ci/fast_test_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 3c84eef4ece4..e5b21615cc63 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -49,7 +49,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( - f"docker run --cap-add=SYS_PTRACE " + f"docker run --cap-add=SYS_PTRACE --timeout 10800 " "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " @@ -58,8 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - # f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 9a338e3ccfd6f02c8863ba00409768746328609e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 11:33:30 -0400 Subject: [PATCH 075/213] added timeout --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index e5b21615cc63..6eba6c0c9a32 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -49,7 +49,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( - f"docker run --cap-add=SYS_PTRACE --timeout 10800 " + f"docker run --cap-add=SYS_PTRACE " "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " @@ -63,7 +63,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume={output_path}:/test_output {image}" + f"--volume={output_path}:/test_output {image} timeout 3h" ) From f93bd634f1a383e316ce228635e6d15348138c20 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 11:40:05 -0400 Subject: [PATCH 076/213] added timeout --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 6eba6c0c9a32..d795a6fc41ea 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -49,7 +49,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( - f"docker run --cap-add=SYS_PTRACE " + f"timeout 3h docker run --cap-add=SYS_PTRACE " "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " @@ -63,7 +63,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume={output_path}:/test_output {image} timeout 3h" + f"--volume={output_path}:/test_output {image}" ) From 51419124abc9337d452fca40d723afd0eb3583a2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 12:36:51 -0400 Subject: [PATCH 077/213] added tag for new runner --- .github/workflows/pull_request.yml | 2 +- tests/ci/fast_test_check.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index aed788e873a8..9699bc80e6a4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -151,7 +151,7 @@ jobs: sudo rm -fr "$TEMP_PATH" FastTest: needs: DockerHubPush - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, style-checker, build] steps: - name: Set envs run: | diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d795a6fc41ea..3fd78e9455be 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,8 +58,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 5ff737f2f5c98fb345466f270425d451609881f8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 12:49:20 -0400 Subject: [PATCH 078/213] pass cred --- tests/ci/fast_test_check.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 3fd78e9455be..e04b0a11d1a8 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,6 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + "-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " + "-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 87bde21fa07a07c44ff583a5d9f1fd26fcf96f2a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 13:08:36 -0400 Subject: [PATCH 079/213] pass cred --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index e04b0a11d1a8..1b385359166f 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,8 +58,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - "-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - "-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + f"-e AWS_ACCESS_KEY_ID=${{AWS_ACCESS_KEY_ID}} " + f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From ce944cc0ca18acde7119310b803c30a101b552b5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 13:15:11 -0400 Subject: [PATCH 080/213] pass cred --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 1b385359166f..d795a6fc41ea 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -58,7 +58,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID=${{AWS_ACCESS_KEY_ID}} " + f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " From 8c24fb3c0721e99bc67c7ae6181d4785549f94cf Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 18:21:57 -0400 Subject: [PATCH 081/213] pass cred vault --- tests/ci/fast_test_check.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d795a6fc41ea..058561024a83 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -46,6 +46,9 @@ # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) +# getting values from vault +aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") +aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( @@ -58,8 +61,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={AWS_ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={AWS_SECRET_ACCESS_KEY} " + f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 37183b254db97c1f9abeb19154f463928cbfa62f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 18:42:36 -0400 Subject: [PATCH 082/213] updated version --- tests/ci/get_robot_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 8f49f29fa971..c798e17ecfae 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -22,7 +22,7 @@ def get_parameter_from_ssm(name, decrypt=True, client=None): if VAULT_URL: if not client: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameter = client.secrets.kv.v2.read_secret_version( + parameter = client.secrets.kv.v4.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"][name] else: @@ -47,7 +47,7 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): if VAULT_URL: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameters = client.secrets.kv.v2.read_secret_version( + parameters = client.secrets.kv.v4.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"] parameters = { From 6222acc3fdec492a1e6704e80e43a4e16afe23a6 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 19:31:34 -0400 Subject: [PATCH 083/213] updated version to v1 --- tests/ci/get_robot_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index c798e17ecfae..88ed80fe6b70 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -22,7 +22,7 @@ def get_parameter_from_ssm(name, decrypt=True, client=None): if VAULT_URL: if not client: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameter = client.secrets.kv.v4.read_secret_version( + parameter = client.secrets.kv.v1.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"][name] else: @@ -47,7 +47,7 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): if VAULT_URL: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameters = client.secrets.kv.v4.read_secret_version( + parameters = client.secrets.kv.v1.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"] parameters = { From b0818a5d2d3fe942b62ff8abc9691bc351e5e521 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 19:38:09 -0400 Subject: [PATCH 084/213] updated version to v2 --- tests/ci/get_robot_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 88ed80fe6b70..8f49f29fa971 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -22,7 +22,7 @@ def get_parameter_from_ssm(name, decrypt=True, client=None): if VAULT_URL: if not client: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameter = client.secrets.kv.v1.read_secret_version( + parameter = client.secrets.kv.v2.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"][name] else: @@ -47,7 +47,7 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): if VAULT_URL: client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) - parameters = client.secrets.kv.v1.read_secret_version( + parameters = client.secrets.kv.v2.read_secret_version( mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH )["data"]["data"] parameters = { From 2d6f81ada92ea43a94e086074c83de394be1dc7a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:12:35 -0400 Subject: [PATCH 085/213] testing for not printing --- tests/ci/fast_test_check.py | 53 ++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 058561024a83..75bd9d8fe5c5 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -47,28 +47,61 @@ csv.field_size_limit(sys.maxsize) # getting values from vault + aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): - return ( +# return ( +# f"timeout 3h docker run --cap-add=SYS_PTRACE " +# "--network=host " # required to get access to IAM credentials +# f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " +# f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " +# f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " +# f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " +# f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " +# f"-e SCCACHE_S3_USE_SSL=true " +# f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " +# # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " +# # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " +# f"-e SCCACHE_REGION={S3_REGION} " +# f"-e SCCACHE_ENDPOINT={S3_URL} " +# f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " +# f"--volume={output_path}:/test_output {image}" +# ) + + env_vars = { + "AWS_ACCESS_KEY_ID": aws_access_key_id, + "AWS_SECRET_ACCESS_KEY": aws_secret_access_key + } + + cmd = ( f"timeout 3h docker run --cap-add=SYS_PTRACE " - "--network=host " # required to get access to IAM credentials - f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " - f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + f"--network=host " + f"-e FASTTEST_WORKSPACE=/fasttest-workspace " + f"-e FASTTEST_OUTPUT=/test_output " + f"-e FASTTEST_SOURCE=/ClickHouse " + f"--cap-add=SYS_PTRACE " f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " - f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + f"-e PULL_REQUEST_NUMBER={pr_number} " + f"-e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " - f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} " + f"-e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " - f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume={output_path}:/test_output {image}" + f"--volume={workspace}:/fasttest-workspace " + f"--volume={repo_path}:/ClickHouse " + f"--volume={output_path}:/test_output " + f"{image}" ) + for key, value in env_vars.items(): + cmd.insert(-1, "-e") + cmd.insert(-1, f"{key}={value}") + + return " ".join(cmd) def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults From c4d984f8fa664ad2af0fceeb2e1682cc1ed8334f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:14:09 -0400 Subject: [PATCH 086/213] testing for not printing --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 75bd9d8fe5c5..f174f0543428 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -51,6 +51,7 @@ aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") + def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): # return ( # f"timeout 3h docker run --cap-add=SYS_PTRACE " From de644c883bd75d0cb07e0f1584b8b48312718f05 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:19:45 -0400 Subject: [PATCH 087/213] testing --- tests/ci/fast_test_check.py | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index f174f0543428..d90f549e607b 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -76,27 +76,27 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i "AWS_SECRET_ACCESS_KEY": aws_secret_access_key } - cmd = ( - f"timeout 3h docker run --cap-add=SYS_PTRACE " - f"--network=host " - f"-e FASTTEST_WORKSPACE=/fasttest-workspace " - f"-e FASTTEST_OUTPUT=/test_output " - f"-e FASTTEST_SOURCE=/ClickHouse " - f"--cap-add=SYS_PTRACE " - f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " - f"-e PULL_REQUEST_NUMBER={pr_number} " - f"-e COMMIT_SHA={commit_sha} " - f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - f"-e SCCACHE_S3_USE_SSL=true " - f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} " - f"-e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e SCCACHE_REGION={S3_REGION} " - f"-e SCCACHE_ENDPOINT={S3_URL} " - f"--volume={workspace}:/fasttest-workspace " - f"--volume={repo_path}:/ClickHouse " - f"--volume={output_path}:/test_output " + cmd = [ + "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", + "--network=host", + "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", + "-e", "FASTTEST_OUTPUT=/test_output", + "-e", "FASTTEST_SOURCE=/ClickHouse", + "--cap-add=SYS_PTRACE", + "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", + "-e", f"PULL_REQUEST_NUMBER={pr_number}", + "-e", f"COMMIT_SHA={commit_sha}", + "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", + "-e", "SCCACHE_S3_USE_SSL=true", + "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", + "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", + "-e", f"SCCACHE_REGION={S3_REGION}", + "-e", f"SCCACHE_ENDPOINT={S3_URL}", + "--volume", f"{workspace}:/fasttest-workspace", + "--volume", f"{repo_path}:/ClickHouse", + "--volume", f"{output_path}:/test_output", f"{image}" - ) + ] for key, value in env_vars.items(): cmd.insert(-1, "-e") From c72b49fc36b3f074da457531e8ac01fb52e6abb5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:27:08 -0400 Subject: [PATCH 088/213] testing --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d90f549e607b..8e2f77f9e2c2 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -72,8 +72,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i # ) env_vars = { - "AWS_ACCESS_KEY_ID": aws_access_key_id, - "AWS_SECRET_ACCESS_KEY": aws_secret_access_key + "AWS_ACCESS_KEY_ID": "***", + "AWS_SECRET_ACCESS_KEY": "***" } cmd = [ From 0c0db81749b4d728261c728e3f05cab2bec26893 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:37:28 -0400 Subject: [PATCH 089/213] testing for creds --- tests/ci/fast_test_check.py | 113 ++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 8e2f77f9e2c2..e975d9ff1edf 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -53,56 +53,81 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): -# return ( -# f"timeout 3h docker run --cap-add=SYS_PTRACE " -# "--network=host " # required to get access to IAM credentials -# f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " -# f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " -# f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " -# f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " -# f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " -# f"-e SCCACHE_S3_USE_SSL=true " -# f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " -# # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " -# # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " -# f"-e SCCACHE_REGION={S3_REGION} " -# f"-e SCCACHE_ENDPOINT={S3_URL} " -# f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " -# f"--volume={output_path}:/test_output {image}" -# ) - - env_vars = { - "AWS_ACCESS_KEY_ID": "***", - "AWS_SECRET_ACCESS_KEY": "***" - } + # return ( + # f"timeout 3h docker run --cap-add=SYS_PTRACE " + # "--network=host " # required to get access to IAM credentials + # f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " + # f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + # f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " + # f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + # f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + # f"-e SCCACHE_S3_USE_SSL=true " + # f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + # # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + # # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + # f"-e SCCACHE_REGION={S3_REGION} " + # f"-e SCCACHE_ENDPOINT={S3_URL} " + # f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + # f"--volume={output_path}:/test_output {image}" + # ) cmd = [ - "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", - "--network=host", - "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", - "-e", "FASTTEST_OUTPUT=/test_output", - "-e", "FASTTEST_SOURCE=/ClickHouse", - "--cap-add=SYS_PTRACE", - "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", - "-e", f"PULL_REQUEST_NUMBER={pr_number}", - "-e", f"COMMIT_SHA={commit_sha}", - "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", - "-e", "SCCACHE_S3_USE_SSL=true", - "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", - "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", - "-e", f"SCCACHE_REGION={S3_REGION}", - "-e", f"SCCACHE_ENDPOINT={S3_URL}", - "--volume", f"{workspace}:/fasttest-workspace", - "--volume", f"{repo_path}:/ClickHouse", - "--volume", f"{output_path}:/test_output", - f"{image}" + f"timeout 3h docker run --cap-add=SYS_PTRACE " + "--network=host " # required to get access to IAM credentials + f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " + f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " + f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + f"-e SCCACHE_S3_USE_SSL=true " + f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + f"-e SCCACHE_REGION={S3_REGION} " + f"-e SCCACHE_ENDPOINT={S3_URL} " + f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume={output_path}:/test_output {image}" ] - for key, value in env_vars.items(): - cmd.insert(-1, "-e") - cmd.insert(-1, f"{key}={value}") + # Generate a masked version of the command for printing + masked_cmd = list(cmd) + for i, arg in enumerate(masked_cmd): + if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): + masked_cmd[i + 1] = "***" return " ".join(cmd) + # env_vars = { + # "AWS_ACCESS_KEY_ID": "***", + # "AWS_SECRET_ACCESS_KEY": "***" + # } + + # cmd = [ + # "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", + # "--network=host", + # "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", + # "-e", "FASTTEST_OUTPUT=/test_output", + # "-e", "FASTTEST_SOURCE=/ClickHouse", + # "--cap-add=SYS_PTRACE", + # "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", + # "-e", f"PULL_REQUEST_NUMBER={pr_number}", + # "-e", f"COMMIT_SHA={commit_sha}", + # "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", + # "-e", "SCCACHE_S3_USE_SSL=true", + # "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", + # "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", + # "-e", f"SCCACHE_REGION={S3_REGION}", + # "-e", f"SCCACHE_ENDPOINT={S3_URL}", + # "--volume", f"{workspace}:/fasttest-workspace", + # "--volume", f"{repo_path}:/ClickHouse", + # "--volume", f"{output_path}:/test_output", + # f"{image}" + # ] + + # for key, value in env_vars.items(): + # cmd.insert(-1, "-e") + # cmd.insert(-1, f"{key}={value}") + + # return " ".join(cmd) def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults From 6b07ede041e9c5bd8dacff27a60c08e3aa468092 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:46:49 -0400 Subject: [PATCH 090/213] testing for creds --- tests/ci/fast_test_check.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index e975d9ff1edf..8cb486f02a8b 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -96,38 +96,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i masked_cmd[i + 1] = "***" return " ".join(cmd) - # env_vars = { - # "AWS_ACCESS_KEY_ID": "***", - # "AWS_SECRET_ACCESS_KEY": "***" - # } - - # cmd = [ - # "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", - # "--network=host", - # "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", - # "-e", "FASTTEST_OUTPUT=/test_output", - # "-e", "FASTTEST_SOURCE=/ClickHouse", - # "--cap-add=SYS_PTRACE", - # "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", - # "-e", f"PULL_REQUEST_NUMBER={pr_number}", - # "-e", f"COMMIT_SHA={commit_sha}", - # "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", - # "-e", "SCCACHE_S3_USE_SSL=true", - # "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", - # "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", - # "-e", f"SCCACHE_REGION={S3_REGION}", - # "-e", f"SCCACHE_ENDPOINT={S3_URL}", - # "--volume", f"{workspace}:/fasttest-workspace", - # "--volume", f"{repo_path}:/ClickHouse", - # "--volume", f"{output_path}:/test_output", - # f"{image}" - # ] - - # for key, value in env_vars.items(): - # cmd.insert(-1, "-e") - # cmd.insert(-1, f"{key}={value}") - - # return " ".join(cmd) def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults From bf8f786a85dc94627baaa12f3cbe6a24439b1f54 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 20:53:22 -0400 Subject: [PATCH 091/213] testing for creds --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 8cb486f02a8b..64a4d9346890 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -90,10 +90,10 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i ] # Generate a masked version of the command for printing - masked_cmd = list(cmd) + masked_cmd = cmd.copy() # Create a copy of the cmd list for i, arg in enumerate(masked_cmd): if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): - masked_cmd[i + 1] = "***" + masked_cmd[i] = arg.split("=")[0] + "=***" return " ".join(cmd) From 80b89f592dfa0733f338ad954c771532c6d8e48c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 21:03:28 -0400 Subject: [PATCH 092/213] testing for creds --- tests/ci/fast_test_check.py | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 64a4d9346890..e83ea0015593 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -81,8 +81,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " @@ -93,9 +93,43 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i masked_cmd = cmd.copy() # Create a copy of the cmd list for i, arg in enumerate(masked_cmd): if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): - masked_cmd[i] = arg.split("=")[0] + "=***" - - return " ".join(cmd) + masked_cmd[i + 1] = "***" + + return " ".join(masked_cmd) + + + # env_vars = { + # "AWS_ACCESS_KEY_ID": "***", + # "AWS_SECRET_ACCESS_KEY": "***" + # } + + # cmd = [ + # "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", + # "--network=host", + # "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", + # "-e", "FASTTEST_OUTPUT=/test_output", + # "-e", "FASTTEST_SOURCE=/ClickHouse", + # "--cap-add=SYS_PTRACE", + # "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", + # "-e", f"PULL_REQUEST_NUMBER={pr_number}", + # "-e", f"COMMIT_SHA={commit_sha}", + # "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", + # "-e", "SCCACHE_S3_USE_SSL=true", + # "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", + # "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", + # "-e", f"SCCACHE_REGION={S3_REGION}", + # "-e", f"SCCACHE_ENDPOINT={S3_URL}", + # "--volume", f"{workspace}:/fasttest-workspace", + # "--volume", f"{repo_path}:/ClickHouse", + # "--volume", f"{output_path}:/test_output", + # f"{image}" + # ] + + # for key, value in env_vars.items(): + # cmd.insert(-1, "-e") + # cmd.insert(-1, f"{key}={value}") + + # return " ".join(cmd) def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults From 564f6e193f9e3b141835efc1420aca8f5a4f7419 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 21:08:39 -0400 Subject: [PATCH 093/213] uncommented --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index e83ea0015593..0db670efad9d 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -81,8 +81,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 2adbff143aaf3517dd5cbcb8522937172c9d6f05 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 21:55:08 -0400 Subject: [PATCH 094/213] uncommented --- tests/ci/fast_test_check.py | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 0db670efad9d..b74dba4cb5c9 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -48,30 +48,12 @@ # getting values from vault -aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") -aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") +# aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") +# aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): - # return ( - # f"timeout 3h docker run --cap-add=SYS_PTRACE " - # "--network=host " # required to get access to IAM credentials - # f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " - # f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " - # f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " - # f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " - # f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - # f"-e SCCACHE_S3_USE_SSL=true " - # f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - # # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " - # f"-e SCCACHE_REGION={S3_REGION} " - # f"-e SCCACHE_ENDPOINT={S3_URL} " - # f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - # f"--volume={output_path}:/test_output {image}" - # ) - - cmd = [ + return ( f"timeout 3h docker run --cap-add=SYS_PTRACE " "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " @@ -81,21 +63,39 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" - ] + ) + + # cmd = [ + # f"timeout 3h docker run --cap-add=SYS_PTRACE " + # "--network=host " # required to get access to IAM credentials + # f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " + # f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + # f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " + # f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + # f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + # f"-e SCCACHE_S3_USE_SSL=true " + # f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " + # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " + # f"-e SCCACHE_REGION={S3_REGION} " + # f"-e SCCACHE_ENDPOINT={S3_URL} " + # f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + # f"--volume={output_path}:/test_output {image}" + # ] - # Generate a masked version of the command for printing - masked_cmd = cmd.copy() # Create a copy of the cmd list - for i, arg in enumerate(masked_cmd): - if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): - masked_cmd[i + 1] = "***" + # # Generate a masked version of the command for printing + # masked_cmd = cmd.copy() # Create a copy of the cmd list + # for i, arg in enumerate(masked_cmd): + # if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): + # masked_cmd[i + 1] = "***" - return " ".join(masked_cmd) + # return " ".join(masked_cmd) # env_vars = { From 3606459549e7ea5e6de1de23b74acbabdd349595 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 22:20:13 -0400 Subject: [PATCH 095/213] testing for 1 and 2nd runner --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9699bc80e6a4..aed788e873a8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -151,7 +151,7 @@ jobs: sudo rm -fr "$TEMP_PATH" FastTest: needs: DockerHubPush - runs-on: [self-hosted, style-checker, build] + runs-on: [self-hosted, style-checker] steps: - name: Set envs run: | From 61b4765b3fffc4c925c84fedf08a5f0b16133ee2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 22:31:21 -0400 Subject: [PATCH 096/213] commented region and endpoint --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index b74dba4cb5c9..22cae94b1e2b 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -65,8 +65,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " - f"-e SCCACHE_REGION={S3_REGION} " - f"-e SCCACHE_ENDPOINT={S3_URL} " + # f"-e SCCACHE_REGION={S3_REGION} " + # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From 5c06dc2d58489036af56075c3e6048344d05dda4 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:00:23 -0400 Subject: [PATCH 097/213] set credentials on vm --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 22cae94b1e2b..b74dba4cb5c9 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -65,8 +65,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " - # f"-e SCCACHE_REGION={S3_REGION} " - # f"-e SCCACHE_ENDPOINT={S3_URL} " + f"-e SCCACHE_REGION={S3_REGION} " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From 6abe65a8ed4c6b3c0b5267d73e16e80487e10da9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:18:26 -0400 Subject: [PATCH 098/213] ibmcloud login on vm --- tests/ci/fast_test_check.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index b74dba4cb5c9..9c76c014baa5 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,8 +63,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 82224b2b6054c8672e14d9d1db196c1c4fe7b733 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:24:28 -0400 Subject: [PATCH 099/213] ibmcloud login on vm --- tests/ci/fast_test_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 9c76c014baa5..a0ce2ad02414 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -46,7 +46,6 @@ # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) -# getting values from vault # aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") # aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") From 8e17aacf996a15c4ecb476c89141cd8d9dd1507c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:36:20 -0400 Subject: [PATCH 100/213] ibmcloud login on vm --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index a0ce2ad02414..9c76c014baa5 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -46,6 +46,7 @@ # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) +# getting values from vault # aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") # aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") From 4f3ffab50cb5dc92933c1282472e18926721348b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:42:31 -0400 Subject: [PATCH 101/213] commenting SCCACHE --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 9c76c014baa5..449625e70560 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,8 +63,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e SCCACHE_REGION={S3_REGION} " - f"-e SCCACHE_ENDPOINT={S3_URL} " + # f"-e SCCACHE_REGION={S3_REGION} " + # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From baed514c964d645df40050a51a082ebfa10e37e4 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 12 Jun 2023 23:52:21 -0400 Subject: [PATCH 102/213] commenting SCCACHE --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 449625e70560..c65e50955b6d 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -61,7 +61,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - f"-e SCCACHE_S3_USE_SSL=true " + # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " From 53470abf44a30b25adc6d9ed3e7802f8d99b744a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:07:47 -0400 Subject: [PATCH 103/213] check for secret --- .github/workflows/pull_request.yml | 3 +++ tests/ci/fast_test_check.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index aed788e873a8..6afda77c18e7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -170,6 +170,9 @@ jobs: name: changed_images path: ${{ env.TEMP_PATH }} - name: Fast Test + env: + ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} + SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index c65e50955b6d..ad3f93b7e892 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,6 +63,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e AWS_ACCESS_KEY_ID=${{ secrets.ACCESS_KEY_ID }} " + f"-e AWS_SECRET_ACCESS_KEY=${{ secrets.SECRET_ACCESS_KEY }} " # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 61510a0cc35054a46151a22d470ea4bfba7e3bdc Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:24:17 -0400 Subject: [PATCH 104/213] check for secret --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ad3f93b7e892..573f18095fee 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,8 +63,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID=${{ secrets.ACCESS_KEY_ID }} " - f"-e AWS_SECRET_ACCESS_KEY=${{ secrets.SECRET_ACCESS_KEY }} " + f"-e AWS_ACCESS_KEY_ID=${{ env.ACCESS_KEY_ID }} " + f"-e AWS_SECRET_ACCESS_KEY=${{ env.SECRET_ACCESS_KEY }} " # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From 26460a079fd5859a3343e9ab39abb4f687b9cff7 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:33:05 -0400 Subject: [PATCH 105/213] check for secret --- .github/workflows/pull_request.yml | 6 +++--- tests/ci/fast_test_check.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6afda77c18e7..82637c64547d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -170,9 +170,9 @@ jobs: name: changed_images path: ${{ env.TEMP_PATH }} - name: Fast Test - env: - ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} + # env: + # ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} + # SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 573f18095fee..d57cdfb44fff 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,8 +63,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID=${{ env.ACCESS_KEY_ID }} " - f"-e AWS_SECRET_ACCESS_KEY=${{ env.SECRET_ACCESS_KEY }} " + f"-e AWS_ACCESS_KEY_ID=${{ secrets.ACCESS_KEY_ID }} " + f"-e AWS_SECRET_ACCESS_KEY=${{secrets.SECRET_ACCESS_KEY }} " # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From bff209bcaa413eb1a9a385e8d8415cff72e53328 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:39:47 -0400 Subject: [PATCH 106/213] check for secret --- .github/workflows/pull_request.yml | 6 +++--- tests/ci/fast_test_check.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 82637c64547d..6afda77c18e7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -170,9 +170,9 @@ jobs: name: changed_images path: ${{ env.TEMP_PATH }} - name: Fast Test - # env: - # ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} - # SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} + env: + ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} + SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d57cdfb44fff..d7f5b4ceb9cf 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -51,6 +51,8 @@ # aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") # aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") +ACCESS_KEY_ID = os.environ.get('ACCESS_KEY_ID') +SECRET_ACCESS_KEY = os.environ.get('SECRET_ACCESS_KEY') def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( @@ -63,8 +65,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID=${{ secrets.ACCESS_KEY_ID }} " - f"-e AWS_SECRET_ACCESS_KEY=${{secrets.SECRET_ACCESS_KEY }} " + f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " + f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " From acd2ef3c352a10d7ded73f63e50830411dc69749 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:44:09 -0400 Subject: [PATCH 107/213] adding secret --- tests/ci/fast_test_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index d7f5b4ceb9cf..fcf3cabb98b1 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -63,7 +63,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - # f"-e SCCACHE_S3_USE_SSL=true " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " From f9f34b825f34c7996df123ef6010e84dedd6f26f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 07:56:02 -0400 Subject: [PATCH 108/213] uncommenetded region n endpoint --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index fcf3cabb98b1..0c65836b7bca 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -66,8 +66,8 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " - # f"-e SCCACHE_REGION={S3_REGION} " - # f"-e SCCACHE_ENDPOINT={S3_URL} " + f"-e SCCACHE_REGION={S3_REGION} " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output {image}" ) From cf614a255368c30834e82bba446f9ca1d98e3680 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 08:14:43 -0400 Subject: [PATCH 109/213] corrected style --- tests/ci/fast_test_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 0c65836b7bca..eb3ed625b14d 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -51,8 +51,8 @@ # aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") # aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") -ACCESS_KEY_ID = os.environ.get('ACCESS_KEY_ID') -SECRET_ACCESS_KEY = os.environ.get('SECRET_ACCESS_KEY') +ACCESS_KEY_ID = os.environ.get("ACCESS_KEY_ID") +SECRET_ACCESS_KEY = os.environ.get("SECRET_ACCESS_KEY") def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( @@ -98,7 +98,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i # return " ".join(masked_cmd) - # env_vars = { # "AWS_ACCESS_KEY_ID": "***", # "AWS_SECRET_ACCESS_KEY": "***" @@ -132,6 +131,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i # return " ".join(cmd) + def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults additional_files = [] From 879ec50ff5aa6efd22e6e54abf5f9e36ac73a824 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 08:15:15 -0400 Subject: [PATCH 110/213] corrected style --- tests/ci/fast_test_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index eb3ed625b14d..99ac4a0823d5 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -54,6 +54,7 @@ ACCESS_KEY_ID = os.environ.get("ACCESS_KEY_ID") SECRET_ACCESS_KEY = os.environ.get("SECRET_ACCESS_KEY") + def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( f"timeout 3h docker run --cap-add=SYS_PTRACE " From 60f22d5e5c4d208d6887ee4cf0162723787d1661 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 08:22:38 -0400 Subject: [PATCH 111/213] removed commented lines --- tests/ci/fast_test_check.py | 66 ------------------------------------- 1 file changed, 66 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 99ac4a0823d5..63c11f5787f0 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -29,8 +29,6 @@ DOCKER_USER, DOCKER_REPO, S3_REGION, - AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, S3_URL, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -46,11 +44,6 @@ # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) -# getting values from vault - -# aws_access_key_id = get_parameter_from_ssm("AWS_ACCESS_KEY_ID") -# aws_secret_access_key = get_parameter_from_ssm("AWS_SECRET_ACCESS_KEY") - ACCESS_KEY_ID = os.environ.get("ACCESS_KEY_ID") SECRET_ACCESS_KEY = os.environ.get("SECRET_ACCESS_KEY") @@ -73,65 +66,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"--volume={output_path}:/test_output {image}" ) - # cmd = [ - # f"timeout 3h docker run --cap-add=SYS_PTRACE " - # "--network=host " # required to get access to IAM credentials - # f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " - # f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " - # f"-e FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache' " - # f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " - # f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " - # f"-e SCCACHE_S3_USE_SSL=true " - # f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={aws_access_key_id} " - # f"-e AWS_SECRET_ACCESS_KEY={aws_secret_access_key} " - # f"-e SCCACHE_REGION={S3_REGION} " - # f"-e SCCACHE_ENDPOINT={S3_URL} " - # f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - # f"--volume={output_path}:/test_output {image}" - # ] - - # # Generate a masked version of the command for printing - # masked_cmd = cmd.copy() # Create a copy of the cmd list - # for i, arg in enumerate(masked_cmd): - # if arg.startswith("-e") and ("AWS_ACCESS_KEY_ID" in arg or "AWS_SECRET_ACCESS_KEY" in arg): - # masked_cmd[i + 1] = "***" - - # return " ".join(masked_cmd) - - # env_vars = { - # "AWS_ACCESS_KEY_ID": "***", - # "AWS_SECRET_ACCESS_KEY": "***" - # } - - # cmd = [ - # "timeout", "3h", "docker", "run", "--cap-add=SYS_PTRACE", - # "--network=host", - # "-e", "FASTTEST_WORKSPACE=/fasttest-workspace", - # "-e", "FASTTEST_OUTPUT=/test_output", - # "-e", "FASTTEST_SOURCE=/ClickHouse", - # "--cap-add=SYS_PTRACE", - # "-e", "FASTTEST_CMAKE_FLAGS='-DCOMPILER_CACHE=sccache'", - # "-e", f"PULL_REQUEST_NUMBER={pr_number}", - # "-e", f"COMMIT_SHA={commit_sha}", - # "-e", "COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1", - # "-e", "SCCACHE_S3_USE_SSL=true", - # "-e", f"SCCACHE_BUCKET={S3_BUILDS_BUCKET}", - # "-e", "SCCACHE_S3_KEY_PREFIX=ccache/sccache", - # "-e", f"SCCACHE_REGION={S3_REGION}", - # "-e", f"SCCACHE_ENDPOINT={S3_URL}", - # "--volume", f"{workspace}:/fasttest-workspace", - # "--volume", f"{repo_path}:/ClickHouse", - # "--volume", f"{output_path}:/test_output", - # f"{image}" - # ] - - # for key, value in env_vars.items(): - # cmd.insert(-1, "-e") - # cmd.insert(-1, f"{key}={value}") - - # return " ".join(cmd) - def process_results(result_folder: str) -> Tuple[str, str, TestResults, List[str]]: test_results = [] # type: TestResults From c73e5283978ba40836c16ac27266447f9b8a4c3e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:04:17 -0400 Subject: [PATCH 112/213] mounted .aws --- .github/workflows/pull_request.yml | 3 --- docker/packager/packager | 2 +- tests/ci/fast_test_check.py | 12 +++++------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6afda77c18e7..aed788e873a8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -170,9 +170,6 @@ jobs: name: changed_images path: ${{ env.TEMP_PATH }} - name: Fast Test - env: - ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py diff --git a/docker/packager/packager b/docker/packager/packager index 30c41ee3b504..40de892a890b 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -104,7 +104,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " - f"{interactive} {image_name}" + f"{interactive} {image_name}" ) logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 63c11f5787f0..08eacdc3af0d 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -44,9 +44,6 @@ # Will help to avoid errors like _csv.Error: field larger than field limit (131072) csv.field_size_limit(sys.maxsize) -ACCESS_KEY_ID = os.environ.get("ACCESS_KEY_ID") -SECRET_ACCESS_KEY = os.environ.get("SECRET_ACCESS_KEY") - def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( @@ -58,11 +55,12 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " - f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " - f"-e SCCACHE_REGION={S3_REGION} " - f"-e SCCACHE_ENDPOINT={S3_URL} " + # f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " + # f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " + # f"-e SCCACHE_REGION={S3_REGION} " + # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"--volume={output_path}:/test_output {image}" ) From 230ea79d3a979dc0bf6a8942c0c30977e3db335e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:23:22 -0400 Subject: [PATCH 113/213] updated comment for run --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 08eacdc3af0d..7bfe19036fd1 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -60,7 +60,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory for creds f"--volume={output_path}:/test_output {image}" ) From 9f9823ac8b106ed3a543ba751c15bd83842fca13 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:30:44 -0400 Subject: [PATCH 114/213] updated comment for run --- tests/ci/fast_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 7bfe19036fd1..08eacdc3af0d 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -60,7 +60,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i # f"-e SCCACHE_REGION={S3_REGION} " # f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory for creds + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"--volume={output_path}:/test_output {image}" ) From 5b305b26766b3e958aa66186f083ccaa56cec661 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:44:04 -0400 Subject: [PATCH 115/213] just mounting creds --- tests/ci/fast_test_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 08eacdc3af0d..42e476ebda02 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -57,10 +57,10 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " # f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " # f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " - # f"-e SCCACHE_REGION={S3_REGION} " - # f"-e SCCACHE_ENDPOINT={S3_URL} " + f"-e SCCACHE_REGION={S3_REGION} " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"--volume={output_path}:/test_output {image}" ) From f394dd0af8a1d54b2267edcef407d1f2b68ab6b1 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:44:41 -0400 Subject: [PATCH 116/213] just mounting creds --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 40de892a890b..30c41ee3b504 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -104,7 +104,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " - f"{interactive} {image_name}" + f"{interactive} {image_name}" ) logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd) From b9f7e97595049dd7a0f9fd6eaccf61da85d8febe Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 10:56:53 -0400 Subject: [PATCH 117/213] just passing endpoint --- tests/ci/fast_test_check.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 42e476ebda02..277412026b52 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -55,9 +55,6 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " - # f"-e AWS_ACCESS_KEY_ID={ACCESS_KEY_ID} " - # f"-e AWS_SECRET_ACCESS_KEY={SECRET_ACCESS_KEY} " - f"-e SCCACHE_REGION={S3_REGION} " f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory From 1ecf10dea23900f908daa8a25587997363c58156 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 11:34:54 -0400 Subject: [PATCH 118/213] removed S3_region --- tests/ci/fast_test_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 277412026b52..c510e3464245 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -28,7 +28,6 @@ TEMP_PATH, DOCKER_USER, DOCKER_REPO, - S3_REGION, S3_URL, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm From 8f02a84c54a8c1cb4ca7bb400d10804a029e7338 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 12:59:37 -0400 Subject: [PATCH 119/213] testing for BuilderDeb release --- docker/packager/packager | 1 + tests/ci/build_check.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docker/packager/packager b/docker/packager/packager index 30c41ee3b504..5be3d78ca71e 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -104,6 +104,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" ) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index fc9cc4170cf6..dfa74f359ce1 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -25,6 +25,7 @@ TEMP_PATH, DOCKER_USER, DOCKER_REPO, + S3_URL, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -65,6 +66,7 @@ def get_packager_cmd( cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1" cmd = ( f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--package-type={package_type} --compiler={comp}" ) From d49f151288e412eccaac3b84b04f99b376eab6d6 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 12:59:43 -0400 Subject: [PATCH 120/213] testing for BuilderDeb release --- .github/workflows/pull_request.yml | 215 +++++++++++++++-------------- 1 file changed, 108 insertions(+), 107 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index aed788e873a8..4acf9de7dfd7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -114,71 +114,71 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, style-checker] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{ runner.temp }}/style_check - ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # StyleCheck: + # needs: DockerHubPush + # runs-on: [self-hosted, style-checker] + # # We need additional `&& ! cancelled()` to have the job being able to cancel + # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{ runner.temp }}/style_check + # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test + # run: | + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ## CompatibilityCheckX86: # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] @@ -244,48 +244,49 @@ jobs: # ######################################################################################### # #################################### ORDINARY BUILDS #################################### # ######################################################################################### -# BuilderDebRelease: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_release -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# fetch-depth: 0 # for performance artifact -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" + BuilderDebRelease: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] From 9f9f0b8eb37e0dc897af8a2777f9318baeec797f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 13:27:14 -0400 Subject: [PATCH 121/213] solving circular import --- tests/ci/constants.py | 7 +++++++ tests/ci/env_helper.py | 6 +----- tests/ci/get_robot_token.py | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 tests/ci/constants.py diff --git a/tests/ci/constants.py b/tests/ci/constants.py new file mode 100644 index 000000000000..860337cb8e63 --- /dev/null +++ b/tests/ci/constants.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +import os + +VAULT_PATH = os.getenv("VAULT_PATH") +VAULT_TOKEN = os.getenv("VAULT_TOKEN") +VAULT_URL = os.getenv("VAULT_URL") +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") \ No newline at end of file diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 8aba88c14522..d22930307c1c 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -11,7 +11,7 @@ TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) -IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "secret") +IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "secret") AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "secret") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) @@ -38,10 +38,6 @@ f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) -VAULT_PATH = os.getenv("VAULT_PATH") -VAULT_TOKEN = os.getenv("VAULT_TOKEN") -VAULT_URL = os.getenv("VAULT_URL") -VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 8f49f29fa971..f93291be60d1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -8,7 +8,7 @@ from github import Github from github.AuthenticatedUser import AuthenticatedUser -from env_helper import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT +from constants import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT @dataclass From ed15e41bfe81487250040edb020149832d05514d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 13:39:26 -0400 Subject: [PATCH 122/213] updating param --- tests/ci/build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index dfa74f359ce1..6c1f0c31c0ae 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -66,7 +66,7 @@ def get_packager_cmd( cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1" cmd = ( f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " - f"-e SCCACHE_ENDPOINT={S3_URL} " + f"SCCACHE_ENDPOINT={S3_URL} " f"--package-type={package_type} --compiler={comp}" ) From aec7fee4fae8013d55952b942c4f001b503f3bae Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 14:02:48 -0400 Subject: [PATCH 123/213] removed endpoint --- tests/ci/build_check.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 6c1f0c31c0ae..8a163609e304 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -65,8 +65,7 @@ def get_packager_cmd( comp = build_config["compiler"] cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1" cmd = ( - f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " - f"SCCACHE_ENDPOINT={S3_URL} " + f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " f"--package-type={package_type} --compiler={comp}" ) From 26da715cea1be4268449aaeb8a972d374d1d3f75 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 18:26:43 -0400 Subject: [PATCH 124/213] added endpoint --- docker/packager/packager | 11 ++++++++++- tests/ci/build_check.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 5be3d78ca71e..3a74769f4b8d 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -124,6 +124,7 @@ def parse_env_variables( package_type: str, cache: str, s3_bucket: str, + s3_endpoint: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -275,6 +276,7 @@ def parse_env_variables( cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket}") + result.append(f"SCCACHE_BUCKET={s3_endpoint}") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -400,6 +402,11 @@ def parse_args() -> argparse.Namespace: "--s3-bucket", help="an S3 bucket used for sscache and clang-tidy-cache", ) + parser.add_argument( + "--s3-endpoint", + default="https://s3.amazonaws.com", + help="the custom endpoint URL for the S3-compatible object storage service" + ) parser.add_argument( "--s3-directory", default="ccache", @@ -428,7 +435,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--docker-user", default=argparse.SUPPRESS, help="docker user") parser.add_argument( "--docker-password", default=argparse.SUPPRESS, help="docker password" - ) + ) + args = parser.parse_args() @@ -478,6 +486,7 @@ def main(): args.package_type, args.cache, args.s3_bucket, + args.s3_endpoint, args.s3_directory, args.s3_rw_access, args.clang_tidy, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 8a163609e304..0088a051e0e2 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -79,6 +79,7 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" + cmd += f" --s3-endpoint={S3_URL}" if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" From 7b097112c35367c019e118eeba79111addaf8d7f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 13 Jun 2023 18:43:42 -0400 Subject: [PATCH 125/213] updated SCCACHE_ENDPOINT --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 3a74769f4b8d..091f72791fb9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -276,7 +276,7 @@ def parse_env_variables( cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket}") - result.append(f"SCCACHE_BUCKET={s3_endpoint}") + result.append(f"SCCACHE_ENDPOINT={s3_endpoint}") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" From 4d16d4add36088f89cac3de64711b2edc376978f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 09:18:25 -0400 Subject: [PATCH 126/213] testing for value --- docker/packager/packager | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 091f72791fb9..334b6cde25f8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -496,13 +496,15 @@ def main(): args.with_coverage, args.with_binaries, ) - if args.docker_user: - subprocess.check_output( # pylint: disable=unexpected-keyword-arg - f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", - input=args.docker_password, - encoding="utf-8", - shell=True, - ) + # if args.docker_user: + print(f"docker_repo: {args.docker_repo}") + print(f"docker_user: {args.docker_user}") + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", + input=args.docker_password, + encoding="utf-8", + shell=True, + ) pre_build(args.clickhouse_repo_path, env_prepared) run_docker_image_with_env( image_with_version, From 5d0a73d2245ff690e8da41a304afdbbce99ef645 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 09:32:59 -0400 Subject: [PATCH 127/213] testing for value --- docker/packager/packager | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 334b6cde25f8..75d84de6e167 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -496,15 +496,15 @@ def main(): args.with_coverage, args.with_binaries, ) - # if args.docker_user: - print(f"docker_repo: {args.docker_repo}") - print(f"docker_user: {args.docker_user}") - subprocess.check_output( # pylint: disable=unexpected-keyword-arg - f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", - input=args.docker_password, - encoding="utf-8", - shell=True, - ) + if args.docker_user: + print(f"docker_repo: {args.docker_repo}") + print(f"docker_user: {args.docker_user}") + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", + input=args.docker_password, + encoding="utf-8", + shell=True, + ) pre_build(args.clickhouse_repo_path, env_prepared) run_docker_image_with_env( image_with_version, From 3224fc11a942c555d988aee5ce9f749439a114c8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 09:52:55 -0400 Subject: [PATCH 128/213] restructure --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 75d84de6e167..e728147924ce 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,8 +103,8 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory + f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"{interactive} {image_name}" ) From 64c711348d2460e5cdd7c053387d2cf709b9e5b2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 10:23:37 -0400 Subject: [PATCH 129/213] replacing mount point with creds --- docker/packager/packager | 22 ++++++++++++++++++---- tests/ci/build_check.py | 6 +++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index e728147924ce..62445d275f00 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,7 +103,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory + # f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"{interactive} {image_name}" ) @@ -125,6 +125,8 @@ def parse_env_variables( cache: str, s3_bucket: str, s3_endpoint: str, + aws_access_key: str, + aws_secret_access_key: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -275,8 +277,10 @@ def parse_env_variables( if cache == "sccache": cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md - result.append(f"SCCACHE_BUCKET={s3_bucket}") - result.append(f"SCCACHE_ENDPOINT={s3_endpoint}") + result.append(f"SCCACHE_BUCKET={s3_bucket} ") + result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") + result.append(f"AWS_ACCESS_KEY_ID={aws_access_key} ") + result.append(f"AWS_SECRET_ACCESS_KEY={aws_secret_access_key} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -406,7 +410,15 @@ def parse_args() -> argparse.Namespace: "--s3-endpoint", default="https://s3.amazonaws.com", help="the custom endpoint URL for the S3-compatible object storage service" - ) + ) + parser.add_argument( + "--aws-access-key", + help="an S3 directory prefix used for sscache and clang-tidy-cache", + ) + parser.add_argument( + "--aws-secret-access-key", + help="an S3 directory prefix used for sscache and clang-tidy-cache", + ) parser.add_argument( "--s3-directory", default="ccache", @@ -487,6 +499,8 @@ def main(): args.cache, args.s3_bucket, args.s3_endpoint, + args.aws_access_key + args.aws_secret_access_key args.s3_directory, args.s3_rw_access, args.clang_tidy, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 0088a051e0e2..5b80ff72abef 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -25,7 +25,9 @@ TEMP_PATH, DOCKER_USER, DOCKER_REPO, - S3_URL, + S3_URL, + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -80,6 +82,8 @@ def get_packager_cmd( cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --s3-endpoint={S3_URL}" + cmd += f" --aws-access-key={AWS_ACCESS_KEY_ID}" + cmd += f" --aws-secret-access-key={AWS_SECRET_ACCESS_KEY}" if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" From 2152607b01e66f1fac0620826792291b02ee4994 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 10:34:57 -0400 Subject: [PATCH 130/213] added , --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 62445d275f00..216814ece6ba 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -499,8 +499,8 @@ def main(): args.cache, args.s3_bucket, args.s3_endpoint, - args.aws_access_key - args.aws_secret_access_key + args.aws_access_key, + args.aws_secret_access_key, args.s3_directory, args.s3_rw_access, args.clang_tidy, From 6948cb2d554396179498a6fbc7c577b78ac3bb56 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 10:36:55 -0400 Subject: [PATCH 131/213] corrected value --- tests/ci/env_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index d22930307c1c..938614b18b4c 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -12,8 +12,8 @@ IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") -AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "secret") -AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "secret") +AWS_ACCESS_KEY_ID = os.getenv("ACCESS_KEY_ID", "secret") +AWS_SECRET_ACCESS_KEY = os.getenv("SECRET_ACCESS_KEY", "secret") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") From 532e0ffa43aecc3a54d0ad2da71cc8574b6ec25f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 11:12:09 -0400 Subject: [PATCH 132/213] mounting volume --- docker/packager/packager | 10 +++------- tests/ci/build_check.py | 4 ---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 216814ece6ba..d0e69eb5eebb 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,7 +103,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - # f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory + f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"{interactive} {image_name}" ) @@ -125,8 +125,6 @@ def parse_env_variables( cache: str, s3_bucket: str, s3_endpoint: str, - aws_access_key: str, - aws_secret_access_key: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -279,8 +277,8 @@ def parse_env_variables( # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket} ") result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") - result.append(f"AWS_ACCESS_KEY_ID={aws_access_key} ") - result.append(f"AWS_SECRET_ACCESS_KEY={aws_secret_access_key} ") + # result.append(f"AWS_ACCESS_KEY_ID={aws_access_key} ") + # result.append(f"AWS_SECRET_ACCESS_KEY={aws_secret_access_key} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -499,8 +497,6 @@ def main(): args.cache, args.s3_bucket, args.s3_endpoint, - args.aws_access_key, - args.aws_secret_access_key, args.s3_directory, args.s3_rw_access, args.clang_tidy, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 5b80ff72abef..38933526c23c 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -26,8 +26,6 @@ DOCKER_USER, DOCKER_REPO, S3_URL, - AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -82,8 +80,6 @@ def get_packager_cmd( cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --s3-endpoint={S3_URL}" - cmd += f" --aws-access-key={AWS_ACCESS_KEY_ID}" - cmd += f" --aws-secret-access-key={AWS_SECRET_ACCESS_KEY}" if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" From 035bc96981b03cf12fe1fc76cb69e1593f867793 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 11:39:11 -0400 Subject: [PATCH 133/213] mounting /home/ubuntu/.aws and s3_region --- docker/packager/packager | 10 +++++++++- tests/ci/build_check.py | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index d0e69eb5eebb..906078dd4131 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,7 +103,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"{interactive} {image_name}" ) @@ -125,6 +125,7 @@ def parse_env_variables( cache: str, s3_bucket: str, s3_endpoint: str, + s3_region: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -277,6 +278,7 @@ def parse_env_variables( # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket} ") result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") + result.append(f"SCCACHE_REGION={s3_region} ") # result.append(f"AWS_ACCESS_KEY_ID={aws_access_key} ") # result.append(f"AWS_SECRET_ACCESS_KEY={aws_secret_access_key} ") sccache_dir = "sccache" @@ -409,6 +411,11 @@ def parse_args() -> argparse.Namespace: default="https://s3.amazonaws.com", help="the custom endpoint URL for the S3-compatible object storage service" ) + parser.add_argument( + "--s3-region", + default="us-east-1", + help="the custom endpoint URL for the S3-compatible object storage service" + ) parser.add_argument( "--aws-access-key", help="an S3 directory prefix used for sscache and clang-tidy-cache", @@ -497,6 +504,7 @@ def main(): args.cache, args.s3_bucket, args.s3_endpoint, + args.s3_region, args.s3_directory, args.s3_rw_access, args.clang_tidy, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 38933526c23c..bd8b85c8d759 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -26,6 +26,7 @@ DOCKER_USER, DOCKER_REPO, S3_URL, + S3_REGION, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -80,6 +81,8 @@ def get_packager_cmd( cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --s3-endpoint={S3_URL}" + cmd += f" --s3-region={S3_REGION}" + if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" From cdc2e0dd195b934582b5db7950355ba6bc714c96 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 12:03:50 -0400 Subject: [PATCH 134/213] restructuring --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 906078dd4131..faecdb87030c 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -102,9 +102,9 @@ def run_docker_image_with_env( ccache_mount = "" cmd = ( - f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"{interactive} {image_name}" ) From cce60959bf38710e8969eb669d5290be92d6cfc0 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 13:07:30 -0400 Subject: [PATCH 135/213] check --- .github/workflows/pull_request.yml | 7 ++++++- docker/packager/packager | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4acf9de7dfd7..f7dea9935105 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -52,8 +52,13 @@ jobs: # echo "Testing $dir" # python3 -m unittest discover -s "$dir" -p '*_test.py' # done + env: + # needs: CheckLabels + runs-on: [self-hosted] + steps: + - name: Check out repository code + run: env DockerHubPushAarch64: - # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: diff --git a/docker/packager/packager b/docker/packager/packager index faecdb87030c..f52c883624af 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -102,9 +102,9 @@ def run_docker_image_with_env( ccache_mount = "" cmd = ( - f"docker run --network=host --user={user} --rm {ccache_mount}" + f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"{interactive} {image_name}" ) From d0c46850c37a000bb4a56489f221e3a3e673d128 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 13:15:01 -0400 Subject: [PATCH 136/213] check --- .github/workflows/pull_request.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f7dea9935105..89f1ad43feca 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -52,16 +52,12 @@ jobs: # echo "Testing $dir" # python3 -m unittest discover -s "$dir" -p '*_test.py' # done - env: - # needs: CheckLabels - runs-on: [self-hosted] - steps: - - name: Check out repository code - run: env DockerHubPushAarch64: # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: + - name: test + run: env - name: Check out repository code uses: ClickHouse/checkout@v1 with: From de6f9c80b0fe8f30298a41ba95315eab134a99b8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 13:34:33 -0400 Subject: [PATCH 137/213] joining command --- .github/workflows/pull_request.yml | 2 -- docker/packager/packager | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 89f1ad43feca..eda304c3ac66 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -56,8 +56,6 @@ jobs: # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - - name: test - run: env - name: Check out repository code uses: ClickHouse/checkout@v1 with: diff --git a/docker/packager/packager b/docker/packager/packager index f52c883624af..966072a10ee7 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,8 +103,8 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " - f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} --volume=/home/ubuntu/.aws:/root/.aws " + # f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory f"{interactive} {image_name}" ) From 3704fef9c3af1047cdecffa62c96141b1f8d000b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 15:13:32 -0400 Subject: [PATCH 138/213] testing --- docker/packager/packager | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index 966072a10ee7..5f12afd3f654 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -103,8 +103,8 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" - f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} --volume=/home/ubuntu/.aws:/root/.aws " - # f"--volume=/home/ubuntu/.aws:/root/.aws " # Mount .aws directory + f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" ) From ed66e30c00c1540c1f23f619cc68fd3b14cd5d60 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 17:16:23 -0400 Subject: [PATCH 139/213] update /.aws/credentials --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 5f12afd3f654..b14bd6b5704f 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -104,7 +104,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " - f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory + f"--volume=/home/ubuntu/.aws/credentials:/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" ) From 812699edd7ff5df35355c6771f25ce6e87c2f118 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 19:33:09 -0400 Subject: [PATCH 140/213] commenting region --- docker/packager/packager | 4 +--- tests/ci/build_check.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index b14bd6b5704f..00d0e2e525b8 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -278,9 +278,7 @@ def parse_env_variables( # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket} ") result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") - result.append(f"SCCACHE_REGION={s3_region} ") - # result.append(f"AWS_ACCESS_KEY_ID={aws_access_key} ") - # result.append(f"AWS_SECRET_ACCESS_KEY={aws_secret_access_key} ") + # result.append(f"SCCACHE_REGION={s3_region} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index bd8b85c8d759..2aea883e6bcd 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -66,7 +66,7 @@ def get_packager_cmd( comp = build_config["compiler"] cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1" cmd = ( - f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " + f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " f"--package-type={package_type} --compiler={comp}" ) From 59c6008b7d3b3c19ab4cde351954168b3d9c51ee Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 20:02:22 -0400 Subject: [PATCH 141/213] removed region and added BuilderDebAarch64 --- .github/workflows/pull_request.yml | 85 +++++++++++++++--------------- docker/packager/packager | 12 +---- tests/ci/build_check.py | 2 - 3 files changed, 45 insertions(+), 54 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index eda304c3ac66..cd487229e578 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -327,48 +327,49 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebAarch64: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_aarch64 -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ runner.temp }}/images_path -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# fetch-depth: 0 # for performance artifact -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderDebAsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] diff --git a/docker/packager/packager b/docker/packager/packager index 00d0e2e525b8..d6632c9426fd 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -125,7 +125,6 @@ def parse_env_variables( cache: str, s3_bucket: str, s3_endpoint: str, - s3_region: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -277,8 +276,7 @@ def parse_env_variables( cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket} ") - result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") - # result.append(f"SCCACHE_REGION={s3_region} ") + result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -409,11 +407,6 @@ def parse_args() -> argparse.Namespace: default="https://s3.amazonaws.com", help="the custom endpoint URL for the S3-compatible object storage service" ) - parser.add_argument( - "--s3-region", - default="us-east-1", - help="the custom endpoint URL for the S3-compatible object storage service" - ) parser.add_argument( "--aws-access-key", help="an S3 directory prefix used for sscache and clang-tidy-cache", @@ -501,8 +494,7 @@ def main(): args.package_type, args.cache, args.s3_bucket, - args.s3_endpoint, - args.s3_region, + args.s3_endpoint, args.s3_directory, args.s3_rw_access, args.clang_tidy, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 2aea883e6bcd..0faeb80714e6 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -26,7 +26,6 @@ DOCKER_USER, DOCKER_REPO, S3_URL, - S3_REGION, ) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub @@ -81,7 +80,6 @@ def get_packager_cmd( cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --s3-endpoint={S3_URL}" - cmd += f" --s3-region={S3_REGION}" if "additional_pkgs" in build_config and build_config["additional_pkgs"]: From 69759b3fc1b4590b83fc72dc70956dc6ad9b82af Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 20:47:52 -0400 Subject: [PATCH 142/213] checking for style check --- .github/workflows/pull_request.yml | 324 ++++++++++++++--------------- 1 file changed, 162 insertions(+), 162 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cd487229e578..fd299e2a4d36 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -113,41 +113,41 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - # StyleCheck: - # needs: DockerHubPush - # runs-on: [self-hosted, style-checker] - # # We need additional `&& ! cancelled()` to have the job being able to cancel - # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{ runner.temp }}/style_check - # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{ runner.temp }}/style_check + ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # BuilderDebRelease: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush] + # runs-on: [self-hosted] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -327,49 +327,49 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - # needs: [DockerHubPush, FastTest, StyleCheck] - needs: [DockerHubPush] - runs-on: [self-hosted] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAarch64: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush] + # runs-on: [self-hosted] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderDebAsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -626,47 +626,47 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwin: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwin: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderBinAarch64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] From e2e42d28ce3ffb96b3d0e6fb2cd78026fc0ae7f1 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 21:08:51 -0400 Subject: [PATCH 143/213] fixing style --- docker/packager/packager | 17 ++++++++--------- tests/ci/build_check.py | 1 - tests/ci/constants.py | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index d6632c9426fd..5c464b70f7d9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -276,7 +276,7 @@ def parse_env_variables( cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md result.append(f"SCCACHE_BUCKET={s3_bucket} ") - result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") + result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -403,9 +403,9 @@ def parse_args() -> argparse.Namespace: help="an S3 bucket used for sscache and clang-tidy-cache", ) parser.add_argument( - "--s3-endpoint", - default="https://s3.amazonaws.com", - help="the custom endpoint URL for the S3-compatible object storage service" + "--s3-endpoint", + default="https://s3.amazonaws.com", + help="the custom endpoint URL for the S3-compatible object storage service", ) parser.add_argument( "--aws-access-key", @@ -414,7 +414,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--aws-secret-access-key", help="an S3 directory prefix used for sscache and clang-tidy-cache", - ) + ) parser.add_argument( "--s3-directory", default="ccache", @@ -443,8 +443,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--docker-user", default=argparse.SUPPRESS, help="docker user") parser.add_argument( "--docker-password", default=argparse.SUPPRESS, help="docker password" - ) - + ) args = parser.parse_args() @@ -494,7 +493,7 @@ def main(): args.package_type, args.cache, args.s3_bucket, - args.s3_endpoint, + args.s3_endpoint, args.s3_directory, args.s3_rw_access, args.clang_tidy, @@ -506,7 +505,7 @@ def main(): ) if args.docker_user: print(f"docker_repo: {args.docker_repo}") - print(f"docker_user: {args.docker_user}") + print(f"docker_user: {args.docker_user}") subprocess.check_output( # pylint: disable=unexpected-keyword-arg f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", input=args.docker_password, diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 0faeb80714e6..c7d7ceb2a477 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -81,7 +81,6 @@ def get_packager_cmd( cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" cmd += f" --s3-endpoint={S3_URL}" - if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" diff --git a/tests/ci/constants.py b/tests/ci/constants.py index 860337cb8e63..7f48b77f85a7 100644 --- a/tests/ci/constants.py +++ b/tests/ci/constants.py @@ -4,4 +4,4 @@ VAULT_PATH = os.getenv("VAULT_PATH") VAULT_TOKEN = os.getenv("VAULT_TOKEN") VAULT_URL = os.getenv("VAULT_URL") -VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") \ No newline at end of file +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") From b7e4c7bc82ecd8daafb96aacc3abddc8177ac016 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 21:25:03 -0400 Subject: [PATCH 144/213] added many jobs --- .github/workflows/pull_request.yml | 888 ++++++++++++++--------------- 1 file changed, 443 insertions(+), 445 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index fd299e2a4d36..f0a704e916bd 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -148,36 +148,36 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" - # FastTest: - # needs: DockerHubPush - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/fasttest - # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # EOF - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.TEMP_PATH }} - # - name: Fast Test - # run: | - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + FastTest: + needs: DockerHubPush + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/fasttest + REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} + - name: Fast Test + run: | + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ## CompatibilityCheckX86: # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] @@ -243,349 +243,49 @@ jobs: # ######################################################################################### # #################################### ORDINARY BUILDS #################################### # ######################################################################################### - # BuilderDebRelease: - # # needs: [DockerHubPush, FastTest, StyleCheck] - # needs: [DockerHubPush] - # runs-on: [self-hosted] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_release - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # fetch-depth: 0 # for performance artifact - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" -# BuilderBinRelease: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_release -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderDebAarch64: - # # needs: [DockerHubPush, FastTest, StyleCheck] - # needs: [DockerHubPush] - # runs-on: [self-hosted] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # fetch-depth: 0 # for performance artifact - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebAsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_asan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebUBsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_ubsan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebTsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_tsan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebMsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_msan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebDebug: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_debug -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build -# run: | -# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' -# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' -# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' -# du -hs "$GITHUB_WORKSPACE/contrib" ||: -# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ########################################################################################## -# ##################################### SPECIAL BUILDS ##################################### -# ########################################################################################## -# BuilderBinClangTidy: + BuilderDebRelease: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -596,7 +296,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_tidy +# BUILD_NAME=binary_release # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -626,48 +326,264 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinDarwin: - # needs: [DockerHubPush, FastTest, StyleCheck] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_darwin - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAarch64: + BuilderDebAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_asan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebUBsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_ubsan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebTsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_tsan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebMsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_msan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebDebug: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_debug + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Apply sparse checkout for contrib # in order to check that it doesn't break build + run: | + rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' + git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' + "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' + du -hs "$GITHUB_WORKSPACE/contrib" ||: + find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -678,7 +594,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_aarch64 +# BUILD_NAME=binary_tidy # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -708,7 +624,48 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinFreeBSD: + BuilderBinDarwin: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -719,7 +676,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_freebsd +# BUILD_NAME=binary_aarch64 # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -749,7 +706,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwinAarch64: +# BuilderBinFreeBSD: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -760,7 +717,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin_aarch64 +# BUILD_NAME=binary_freebsd # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -790,6 +747,47 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwinAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderBinPPC64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -916,30 +914,30 @@ jobs: # ############################################################################################ # ##################################### Docker images ####################################### # ############################################################################################ -# DockerServerImages: -# needs: -# - BuilderDebRelease -# - BuilderDebAarch64 -# runs-on: [self-hosted, style-checker] -# steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself -# - name: Check docker clickhouse/clickhouse-server building -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_server.py --release-type head --no-push \ -# --image-repo clickhouse/clickhouse-server --image-path docker/server -# python3 docker_server.py --release-type head --no-push \ -# --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # ############################################################################################ # ##################################### BUILD REPORTER ####################################### # ############################################################################################ From f6a09ed2dff0aa9e455c28f16d5ad4da8d6f6f1c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 21:34:55 -0400 Subject: [PATCH 145/213] style fix --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f0a704e916bd..35b731c92935 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -244,7 +244,7 @@ jobs: # #################################### ORDINARY BUILDS #################################### # ######################################################################################### BuilderDebRelease: - needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: - name: Set envs From fcf8b8ce0fa732d6206fd42d31adc91de1d11864 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Wed, 14 Jun 2023 23:36:54 -0400 Subject: [PATCH 146/213] added yaml for testing --- .github/workflows/pull_test.yaml | 109 +++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/pull_test.yaml diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml new file mode 100644 index 000000000000..d163e4c6cd18 --- /dev/null +++ b/.github/workflows/pull_test.yaml @@ -0,0 +1,109 @@ +name: PullTestCI + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, testing] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, testing] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, testing] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json + BuilderBinDarwinAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, testing] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" \ No newline at end of file From 1e71101bc66714c283ebc965ce660c0b8c05a6a9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 11:14:49 -0400 Subject: [PATCH 147/213] build image for rpm deb --- .github/workflows/pull_test.yaml | 102 ++++++++++++++++------------- docker/test/install/deb/Dockerfile | 1 + docker/test/install/rpm/Dockerfile | 1 + 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index d163e4c6cd18..9db685672646 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -6,9 +6,23 @@ env: on: # yamllint disable-line rule:truthy push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, testing] + runs-on: [self-hosted, test-only] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -24,7 +38,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, testing] + runs-on: [self-hosted, test-only] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -41,7 +55,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, testing] + runs-on: [self-hosted, test-only] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -66,44 +80,44 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - BuilderBinDarwinAarch64: - needs: [DockerHubPush] - runs-on: [self-hosted, testing] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" \ No newline at end of file + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, testing] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" \ No newline at end of file diff --git a/docker/test/install/deb/Dockerfile b/docker/test/install/deb/Dockerfile index 9614473c69b4..3681e276ec8e 100644 --- a/docker/test/install/deb/Dockerfile +++ b/docker/test/install/deb/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:22.04 # The Dockerfile is nicely borrowed from # https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile + ENV \ DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 \ diff --git a/docker/test/install/rpm/Dockerfile b/docker/test/install/rpm/Dockerfile index c55e0fe25072..bbbdf1b168fa 100644 --- a/docker/test/install/rpm/Dockerfile +++ b/docker/test/install/rpm/Dockerfile @@ -3,6 +3,7 @@ FROM centos:8 # The Dockerfile is nicely borrowed from # https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile + ENV \ LANG=C.UTF-8 \ container=docker \ From 7b6f7be7695941a76b01e6548a78b1c596ea3ced Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 11:21:57 -0400 Subject: [PATCH 148/213] testing for run --- .github/workflows/pull_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 9db685672646..7cff26bd7b91 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -5,6 +5,7 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy + push: # pull_request: # types: From 493d26add165fd5caab55b784d4bffdc6c9d4a9d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 11:26:37 -0400 Subject: [PATCH 149/213] testing --- .github/workflows/pull_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 7cff26bd7b91..63c527bda511 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -25,6 +25,8 @@ jobs: DockerHubPushAarch64: runs-on: [self-hosted, test-only] steps: + - name: test + run: env - name: Check out repository code uses: ClickHouse/checkout@v1 with: From d5f9983f39a4ab3cd5d0dd04b71f2ffee49fb9ee Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 17:32:11 -0400 Subject: [PATCH 150/213] checking for install packages --- .github/workflows/pull_test.yaml | 152 ++++++++++++++++++++++++++++++- tests/ci/install_check.py | 15 ++- 2 files changed, 162 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 63c527bda511..a301e4419db4 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -41,7 +41,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, test-only] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -83,6 +83,156 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json + BuilderDebRelease: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + BuilderDebAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + InstallPackagesTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, test-only] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/test_install + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Install packages (amd64) + REPO_COPY=${{runner.temp}}/test_install/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Test packages installation + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 install_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + InstallPackagesTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/test_install + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Install packages (arm64) + REPO_COPY=${{runner.temp}}/test_install/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Test packages installation + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 install_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # BuilderBinDarwinAarch64: # needs: [DockerHubPush] # runs-on: [self-hosted, testing] diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index d619ce96ceec..8b75e3d97869 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -27,8 +27,8 @@ ) from compress_files import compress_fast from docker_pull_helper import get_image_with_version, DockerImage -from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult from s3_helper import S3Helper @@ -37,8 +37,8 @@ from upload_result_helper import upload_results -RPM_IMAGE = "clickhouse/install-rpm-test" -DEB_IMAGE = "clickhouse/install-deb-test" +RPM_IMAGE = "{DOCKER_REPO}/clickhouse/install-rpm-test" +DEB_IMAGE = "{DOCKER_REPO}/clickhouse/install-deb-test" TEMP_PATH = Path(TEMP) LOGS_PATH = TEMP_PATH / "tests_logs" SUCCESS = "success" @@ -267,6 +267,13 @@ def main(): pr_info = PRInfo() + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + if CI: gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) From 83f69ebfdb7cdfc8b1b9a9150e12d00032dd0b33 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 17:33:43 -0400 Subject: [PATCH 151/213] removed dependencies --- .github/workflows/pull_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index a301e4419db4..bf58da141d01 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -126,7 +126,7 @@ jobs: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" BuilderDebAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: - name: Set envs From b86469003c8705eef9062f60b91288184587acc2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 17:36:49 -0400 Subject: [PATCH 152/213] removed ectra lines in Dockerfile --- docker/test/install/deb/Dockerfile | 1 - docker/test/install/rpm/Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/docker/test/install/deb/Dockerfile b/docker/test/install/deb/Dockerfile index 3681e276ec8e..9614473c69b4 100644 --- a/docker/test/install/deb/Dockerfile +++ b/docker/test/install/deb/Dockerfile @@ -3,7 +3,6 @@ FROM ubuntu:22.04 # The Dockerfile is nicely borrowed from # https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile - ENV \ DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 \ diff --git a/docker/test/install/rpm/Dockerfile b/docker/test/install/rpm/Dockerfile index bbbdf1b168fa..c55e0fe25072 100644 --- a/docker/test/install/rpm/Dockerfile +++ b/docker/test/install/rpm/Dockerfile @@ -3,7 +3,6 @@ FROM centos:8 # The Dockerfile is nicely borrowed from # https://github.com/lionelnicolas/docker-ubuntu-systemd/blob/83aa3249146f5df264fe45353f79fc76eb1e42d7/Dockerfile - ENV \ LANG=C.UTF-8 \ container=docker \ From a9bcc20d3dce1f65083f8953addcc98c5ae5b622 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 19:07:20 -0400 Subject: [PATCH 153/213] checking for mark release ready --- .github/workflows/pull_tests.yml | 266 +++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 .github/workflows/pull_tests.yml diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml new file mode 100644 index 000000000000..386496128a41 --- /dev/null +++ b/.github/workflows/pull_tests.yml @@ -0,0 +1,266 @@ +name: PullTestCI + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, test-only] + steps: + - name: test + run: env + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, test-only] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json + BuilderBinDarwin: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwinAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebRelease: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + BuilderDebAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, test-only] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + MarkReleaseReady: + needs: + - BuilderBinDarwin + - BuilderBinDarwinAarch64 + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Mark Commit Release Ready + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 mark_release_ready.py \ No newline at end of file From f303d99a38cdc310306aaef9aa545358082af94e Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 19:30:32 -0400 Subject: [PATCH 154/213] added f string --- tests/ci/install_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 8b75e3d97869..cec89b0b791a 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -37,8 +37,8 @@ from upload_result_helper import upload_results -RPM_IMAGE = "{DOCKER_REPO}/clickhouse/install-rpm-test" -DEB_IMAGE = "{DOCKER_REPO}/clickhouse/install-deb-test" +RPM_IMAGE = f"{DOCKER_REPO}/clickhouse/install-rpm-test" +DEB_IMAGE = f"{DOCKER_REPO}/clickhouse/install-deb-test" TEMP_PATH = Path(TEMP) LOGS_PATH = TEMP_PATH / "tests_logs" SUCCESS = "success" From 40a6b08070c9c4344a97afdfbc784e9f69f53966 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Thu, 15 Jun 2023 19:48:43 -0400 Subject: [PATCH 155/213] added builder --- .github/workflows/pull_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index bf58da141d01..66250ed8f7ce 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -23,7 +23,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, test-only] + runs-on: [self-hosted, builder] steps: - name: test run: env From 75b43637b1e10ce5611227a4554de98ef9a395f8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 16 Jun 2023 00:58:57 -0400 Subject: [PATCH 156/213] checking for fastest image --- .github/workflows/pull_test.yaml | 294 +++++++++++++++++-------------- docker/test/fasttest/Dockerfile | 2 + docker/test/fasttest/run.sh | 11 +- 3 files changed, 171 insertions(+), 136 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 66250ed8f7ce..d953325a0a50 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -23,7 +23,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, builder] + runs-on: [self-hosted, build] steps: - name: test run: env @@ -41,7 +41,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, builder] + runs-on: [self-hosted, build] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -58,7 +58,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, test-only] + runs-on: [self-hosted, build] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -83,156 +83,186 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - BuilderDebRelease: - needs: [DockerHubPush] - runs-on: [self-hosted, builder] + FastTest: + needs: DockerHubPush + runs-on: [self-hosted, build] steps: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse + TEMP_PATH=${{runner.temp}}/fasttest + REPO_COPY=${{runner.temp}}/fasttest/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderDebAarch64: - needs: [DockerHubPush] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - name: Download changed images uses: actions/download-artifact@v3 with: name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build + path: ${{ env.TEMP_PATH }} + - name: Fast Test run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - InstallPackagesTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, test-only] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (amd64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" + cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - name: Cleanup if: always() run: | docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebRelease: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # InstallPackagesTestRelease: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, test-only] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/test_install + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Install packages (amd64) + # REPO_COPY=${{runner.temp}}/test_install/ClickHouse + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Test packages installation + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 install_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # InstallPackagesTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/test_install + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Install packages (arm64) + # REPO_COPY=${{runner.temp}}/test_install/ClickHouse + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Test packages installation + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 install_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # BuilderBinDarwinAarch64: # needs: [DockerHubPush] # runs-on: [self-hosted, testing] diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 4b7ca4709407..92324df63d35 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -4,6 +4,8 @@ ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG +ARG GITHUB_REPOSITORY + ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY RUN apt-get update \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 1b19b787366a..abf47681f2cf 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -11,6 +11,9 @@ stage=${stage:-} # Compiler version, normally set by Dockerfile export LLVM_VERSION=${LLVM_VERSION:-16} +# Access the GITHUB_REPOSITORY environment variable +github_repository=${GITHUB_REPOSITORY} + # A variable to pass additional flags to CMake. # Here we explicitly default it to nothing so that bash doesn't complain about # it being undefined. Also read it as array so that we can pass an empty list @@ -61,8 +64,8 @@ function start_server clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" & set +m - # Add a log message to indicate the server startup process has begun - echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup initiated" >> "$FASTTEST_OUTPUT/server.log" + # # Add a log message to indicate the server startup process has begun + # echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup initiated" >> "$FASTTEST_OUTPUT/server.log" for _ in {1..60}; do if clickhouse-client --query "select 1"; then @@ -81,13 +84,13 @@ function start_server echo "ClickHouse server pid '$server_pid' started and responded" # Add a log message to indicate the server startup process has completed - echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup completed" >> "$FASTTEST_OUTPUT/server.log" + # echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup completed" >> "$FASTTEST_OUTPUT/server.log" } function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/ClibMouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/${github_repository}/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From 8d4912e3a7cbdb4906fc274d97bc369fb8a1e069 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 16 Jun 2023 01:19:40 -0400 Subject: [PATCH 157/213] checking for fastest image --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index abf47681f2cf..31632e7636b8 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -90,7 +90,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/${github_repository}/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/${github_repository}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" From 287abc52f09b1ce4afd27e0b7f8fb5a3d949f8e8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 16 Jun 2023 08:38:53 -0400 Subject: [PATCH 158/213] test for install package --- .github/workflows/pull_test.yaml | 324 +++++++++++++++---------------- 1 file changed, 162 insertions(+), 162 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index d953325a0a50..b13865f2fff2 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -23,7 +23,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, build] + runs-on: [self-hosted, builder] steps: - name: test run: env @@ -41,7 +41,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, build] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -58,7 +58,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, build] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -83,186 +83,186 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - FastTest: - needs: DockerHubPush - runs-on: [self-hosted, build] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderDebRelease: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] + # FastTest: + # needs: DockerHubPush + # runs-on: [self-hosted, build] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_release # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true - # fetch-depth: 0 # for performance artifact - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" - # BuilderDebAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF # - name: Download changed images # uses: actions/download-artifact@v3 # with: # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # fetch-depth: 0 # for performance artifact - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # InstallPackagesTestRelease: - # needs: [BuilderDebRelease] - # runs-on: [self-hosted, test-only] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/test_install - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Install packages (amd64) - # REPO_COPY=${{runner.temp}}/test_install/ClickHouse - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Test packages installation - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 install_check.py "$CHECK_NAME" - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" - # InstallPackagesTestAarch64: - # needs: [BuilderDebAarch64] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/test_install - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Install packages (arm64) - # REPO_COPY=${{runner.temp}}/test_install/ClickHouse - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v3 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Test packages installation - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 install_check.py "$CHECK_NAME" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebRelease: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + BuilderDebAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + InstallPackagesTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, test-only] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/test_install + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Install packages (amd64) + REPO_COPY=${{runner.temp}}/test_install/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Test packages installation + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 install_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + InstallPackagesTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/test_install + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Install packages (arm64) + REPO_COPY=${{runner.temp}}/test_install/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Test packages installation + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 install_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # BuilderBinDarwinAarch64: # needs: [DockerHubPush] # runs-on: [self-hosted, testing] From 2a3e6faf23faba7cc85ff9ba2ab0681dbc1ad4c9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 16 Jun 2023 09:25:57 -0400 Subject: [PATCH 159/213] checking for BuilderBinClangTidy --- .github/workflows/pull_tests.yml | 329 +++++++++++++++++-------------- 1 file changed, 185 insertions(+), 144 deletions(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 386496128a41..1d1247ba01ee 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -1,4 +1,4 @@ -name: PullTestCI +name: PullclangCI env: # Force the stdout and stderr streams to be unbuffered @@ -82,7 +82,7 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - BuilderBinDarwin: + BuilderBinClangTidy: needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: @@ -93,7 +93,7 @@ jobs: IMAGES_PATH=${{runner.temp}}/images_path REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin + BUILD_NAME=binary_tidy EOF - name: Download changed images uses: actions/download-artifact@v3 @@ -123,144 +123,185 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebRelease: - needs: [DockerHubPush] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderDebAarch64: - needs: [DockerHubPush] - runs-on: [self-hosted, test-only] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - MarkReleaseReady: - needs: - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Mark Commit Release Ready - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 mark_release_ready.py \ No newline at end of file + # BuilderBinDarwin: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebRelease: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, test-only] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # MarkReleaseReady: + # needs: + # - BuilderBinDarwin + # - BuilderBinDarwinAarch64 + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Mark Commit Release Ready + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 mark_release_ready.py \ No newline at end of file From 37ee60de0d81637bb69c42c3d38541a44815bf3c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 17 Jun 2023 19:58:50 -0400 Subject: [PATCH 160/213] testing for functional --- .github/workflows/pull_test.yaml | 4825 +++++++++++++++++++++++++++++- 1 file changed, 4723 insertions(+), 102 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index b13865f2fff2..bd48711b884d 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -5,7 +5,6 @@ env: PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy - push: # pull_request: # types: @@ -20,13 +19,44 @@ on: # yamllint disable-line rule:truthy # - 'SECURITY.md' # - 'docker/docs/**' # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' + # - 'utils/check-style/aspell-ignore/**' +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done DockerHubPushAarch64: - runs-on: [self-hosted, builder] + # needs: CheckLabels + runs-on: [self-hosted, test-only] steps: - - name: test - run: env - name: Check out repository code uses: ClickHouse/checkout@v1 with: @@ -41,7 +71,8 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, builder] + # needs: CheckLabels + runs-on: [self-hosted, test-only] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -83,38 +114,138 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - # FastTest: - # needs: DockerHubPush - # runs-on: [self-hosted, build] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/fasttest - # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # EOF - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.TEMP_PATH }} - # - name: Fast Test - # run: | - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + StyleCheck: + needs: DockerHubPush + runs-on: [self-hosted, builder] + # We need additional `&& ! cancelled()` to have the job being able to cancel + if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{ runner.temp }}/style_check + ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/fasttest + REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + EOF + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} + - name: Fast Test + run: | + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +## CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### BuilderDebRelease: - needs: [DockerHubPush] + needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: - name: Set envs @@ -155,8 +286,49 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: - needs: [DockerHubPush] + needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: - name: Set envs @@ -197,110 +369,4559 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - InstallPackagesTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, test-only] + BuilderDebAsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] steps: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (amd64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_asan EOF - - name: Download json reports + - name: Download changed images uses: actions/download-artifact@v3 with: - path: ${{ env.REPORTS_PATH }} + name: changed_images + path: ${{ env.IMAGES_PATH }} - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - - name: Test packages installation + submodules: true + - name: Build run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - name: Cleanup if: always() run: | docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebUBsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_ubsan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | sudo rm -fr "$TEMP_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebTsan: + needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_tsan EOF - - name: Download json reports + - name: Download changed images uses: actions/download-artifact@v3 with: - path: ${{ env.REPORTS_PATH }} + name: changed_images + path: ${{ env.IMAGES_PATH }} - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - - name: Test packages installation + submodules: true + - name: Build run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - name: Cleanup if: always() run: | docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebMsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_msan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | sudo rm -fr "$TEMP_PATH" - # BuilderBinDarwinAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, testing] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_darwin_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" \ No newline at end of file + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebDebug: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_debug + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Apply sparse checkout for contrib # in order to check that it doesn't break build + run: | + rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' + git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' + "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' + du -hs "$GITHUB_WORKSPACE/contrib" ||: + find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwin: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwinAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: +# needs: +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# InstallPackagesTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## + FunctionalStatelessTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (aarch64) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (asan) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (asan) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestUBsan0: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (ubsan) + REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan1: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan0: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan1: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan2: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug0: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug1: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug2: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## + FunctionalStatefulTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (release) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (aarch64) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (asan) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestTsan: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (tsan) + REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_msan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (msan) + REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (ubsan) + REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (debug) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## +# StressTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (asan) +# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestTsan: +# needs: [BuilderDebTsan] +# # func testers have 16 cores + 128 GB memory +# # while stress testers have 36 cores + 72 memory +# # It would be better to have something like 32 + 128, +# # but such servers almost unavailable as spot instances. +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (tsan) +# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (msan) +# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_undefined +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (ubsan) +# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (debug) +# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# +# IntegrationTestsAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" \ No newline at end of file From 81934d9d6c18db4795f3b04cdfcf3c26051d244b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 17 Jun 2023 20:15:20 -0400 Subject: [PATCH 161/213] generalizing functional test --- tests/ci/functional_test_check.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 037bb13f1f87..55927886fa6e 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -30,8 +30,8 @@ ) from docker_pull_helper import get_image_with_version from download_release_packages import download_last_release -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -66,9 +66,9 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): def get_image_name(check_name): if "stateless" in check_name.lower(): - return "clickhouse/stateless-test" + return f"{DOCKER_REPO}/clickhouse/stateless-test" if "stateful" in check_name.lower(): - return "clickhouse/stateful-test" + return f"{DOCKER_REPO}/clickhouse/stateful-test" else: raise Exception(f"Cannot deduce image name based on check name {check_name}") @@ -304,6 +304,13 @@ def main(): ) sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + image_name = get_image_name(check_name) docker_image = get_image_with_version(reports_path, image_name) From 46df90633e912b50d8f8e8392bba9b465c7a39d9 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 17 Jun 2023 20:20:07 -0400 Subject: [PATCH 162/213] commenting style-check --- .github/workflows/pull_test.yaml | 70 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index bd48711b884d..4e21af10ca56 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -114,41 +114,41 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, builder] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{ runner.temp }}/style_check - ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{ runner.temp }}/style_check + # ROBOT_CLICKHOUSE_SSH_KEY< Date: Sat, 17 Jun 2023 20:21:58 -0400 Subject: [PATCH 163/213] changing style-check --- .github/workflows/pull_test.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 4e21af10ca56..eb9e6ddeda05 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -114,13 +114,14 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - # StyleCheck: - # needs: DockerHubPush - # runs-on: [self-hosted, builder] - # # We need additional `&& ! cancelled()` to have the job being able to cancel - # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - # steps: - # - name: Set envs + StyleCheck: + needs: DockerHubPush + runs-on: [self-hosted, builder] + # We need additional `&& ! cancelled()` to have the job being able to cancel + if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + steps: + - name: Set envs + run: echo "hi" # run: | # cat >> "$GITHUB_ENV" << 'EOF' # TEMP_PATH=${{ runner.temp }}/style_check @@ -245,7 +246,7 @@ jobs: # #################################### ORDINARY BUILDS #################################### # ######################################################################################### BuilderDebRelease: - needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] steps: - name: Set envs From 8be16b68a094aed0862baef50ef29c35c11620c8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 18 Jun 2023 00:14:15 -0400 Subject: [PATCH 164/213] skipping fasttest --- .github/workflows/pull_test.yaml | 51 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index eb9e6ddeda05..fe261b7032fa 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -155,31 +155,32 @@ jobs: runs-on: [self-hosted, builder] steps: - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + run: echo "test" + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test + # run: | + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" ## CompatibilityCheckX86: # needs: [BuilderDebRelease] # runs-on: [self-hosted, style-checker] From dec67604bb9cbdc47e36ee524b4e16cea742f9fc Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 18 Jun 2023 02:27:46 -0400 Subject: [PATCH 165/213] testing run --- .github/workflows/pull_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index fe261b7032fa..5ec11a75299d 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -4,6 +4,7 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + on: # yamllint disable-line rule:truthy push: # pull_request: From 57cf1aabd4a9846428ffd20c5e02203a60344315 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 18 Jun 2023 13:45:32 -0400 Subject: [PATCH 166/213] testing run for stress and integration test --- .github/workflows/pull_test.yaml | 1160 ++++++++++++++-------------- tests/ci/functional_test_check.py | 2 +- tests/ci/integration_test_check.py | 33 +- tests/ci/stress_check.py | 13 +- 4 files changed, 611 insertions(+), 597 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 5ec11a75299d..ddbb87e42d0b 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -3179,17 +3179,189 @@ jobs: # ############################################################################################## # ######################################### STRESS TESTS ####################################### # ############################################################################################## -# StressTestAsan: + StressTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (asan) + REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestTsan: + needs: [BuilderDebTsan] + # func testers have 16 cores + 128 GB memory + # while stress testers have 36 cores + 72 memory + # It would be better to have something like 32 + 128, + # but such servers almost unavailable as spot instances. + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (tsan) + REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (msan) + REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_undefined + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (ubsan) + REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (debug) + REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: # needs: [BuilderDebAsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_asan +# TEMP_PATH=${{runner.temp}}/upgrade_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (asan) -# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3199,34 +3371,31 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Upgrade check # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 upgrade_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestTsan: +# UpgradeCheckTsan: # needs: [BuilderDebTsan] -# # func testers have 16 cores + 128 GB memory -# # while stress testers have 36 cores + 72 memory -# # It would be better to have something like 32 + 128, -# # but such servers almost unavailable as spot instances. +# # same as for stress test with tsan # runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_thread +# TEMP_PATH=${{runner.temp}}/upgrade_thread # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (tsan) -# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3236,30 +3405,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Upgrade check # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 upgrade_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestMsan: +# UpgradeCheckMsan: # needs: [BuilderDebMsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_memory +# TEMP_PATH=${{runner.temp}}/upgrade_memory # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (msan) -# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3269,30 +3438,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Upgrade check # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 upgrade_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestUBsan: -# needs: [BuilderDebUBsan] +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_undefined +# TEMP_PATH=${{runner.temp}}/upgrade_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (ubsan) -# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3302,30 +3471,33 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Upgrade check # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 upgrade_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, stress-tester] +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_debug +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (debug) -# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3335,33 +3507,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Fuzzer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 ast_fuzzer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ######################################### UPGRADE CHECK ###################################### -# ############################################################################################## -# UpgradeCheckAsan: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_asan +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (asan) -# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3371,31 +3540,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Upgrade check +# - name: Fuzzer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" +# python3 ast_fuzzer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UpgradeCheckTsan: -# needs: [BuilderDebTsan] -# # same as for stress test with tsan -# runs-on: [self-hosted, func-tester] +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_thread +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (tsan) -# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3405,30 +3573,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Upgrade check +# - name: Fuzzer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" +# python3 ast_fuzzer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UpgradeCheckMsan: +# ASTFuzzerTestMSan: # needs: [BuilderDebMsan] -# runs-on: [self-hosted, stress-tester] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_memory +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (msan) -# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3438,30 +3606,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Upgrade check +# - name: Fuzzer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" +# python3 ast_fuzzer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UpgradeCheckDebug: +# ASTFuzzerTestDebug: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, stress-tester] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_debug +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (debug) -# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3471,181 +3639,13 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Upgrade check +# - name: Fuzzer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ##################################### AST FUZZERS ############################################ -# ############################################################################################## -# ASTFuzzerTestAsan: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (asan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestTsan: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (tsan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestUBSan: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (ubsan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestMSan: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (msan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (debug) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" +# python3 ast_fuzzer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | @@ -3655,298 +3655,123 @@ jobs: # ############################################################################################# # ############################# INTEGRATION TESTS ############################################# # ############################################################################################# -# IntegrationTestsAsan0: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan1: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan2: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan2: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # IntegrationTestsAsan3: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan4: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan5: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=5 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan0: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan1: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan2: -# needs: [BuilderDebTsan] +# needs: [BuilderDebAsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=2 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -3970,18 +3795,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan3: -# needs: [BuilderDebTsan] +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=3 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -4005,18 +3830,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan4: -# needs: [BuilderDebTsan] +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=4 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -4040,7 +3865,147 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan5: + IntegrationTestsTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan3: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: # needs: [BuilderDebTsan] # runs-on: [self-hosted, stress-tester] # steps: @@ -4051,7 +4016,7 @@ jobs: # REPORTS_PATH=${{runner.temp}}/reports_dir # CHECK_NAME=Integration tests (tsan) # REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_NUM=4 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -4075,54 +4040,19 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease0: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=4 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease1: -# needs: [BuilderDebRelease] +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] # runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4145,6 +4075,76 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" # IntegrationTestsRelease2: # needs: [BuilderDebRelease] # runs-on: [self-hosted, stress-tester] diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 55927886fa6e..3f5891e86b1b 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -309,7 +309,7 @@ def main(): input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, - ) + ) image_name = get_image_name(check_name) docker_image = get_image_with_version(reports_path, image_name) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 8ef6244a1c55..190ad1827808 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -27,8 +27,8 @@ ) from docker_pull_helper import get_images_with_versions from download_release_packages import download_last_release -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -40,17 +40,17 @@ # When update, update # integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too IMAGES = [ - "clickhouse/integration-tests-runner", - "clickhouse/mysql-golang-client", - "clickhouse/mysql-java-client", - "clickhouse/mysql-js-client", - "clickhouse/mysql-php-client", - "clickhouse/postgresql-java-client", - "clickhouse/integration-test", - "clickhouse/kerberos-kdc", - "clickhouse/kerberized-hadoop", - "clickhouse/integration-helper", - "clickhouse/dotnet-client", + f"{DOCKER_REPO}/clickhouse/integration-tests-runner", + f"{DOCKER_REPO}/clickhouse/mysql-golang-client", + f"{DOCKER_REPO}/clickhouse/mysql-java-client", + f"{DOCKER_REPO}/clickhouse/mysql-js-client", + f"{DOCKER_REPO}/clickhouse/mysql-php-client", + f"{DOCKER_REPO}/clickhouse/postgresql-java-client", + f"{DOCKER_REPO}/clickhouse/integration-test", + f"{DOCKER_REPO}/clickhouse/kerberos-kdc", + f"{DOCKER_REPO}/clickhouse/kerberized-hadoop", + f"{DOCKER_REPO}/clickhouse/integration-helper", + f"{DOCKER_REPO}/clickhouse/dotnet-client", ] @@ -206,6 +206,13 @@ def main(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + images = get_images_with_versions(reports_path, IMAGES) images_with_versions = {i.name: i.version for i in images} result_path = os.path.join(temp_path, "output_dir") diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index ac280916a2f7..8b3987bb248a 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -18,8 +18,8 @@ ) from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_image_with_version -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -131,6 +131,13 @@ def run_stress_test(docker_image_name): logging.info("Check is already finished according to github status, exiting") sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + docker_image = get_image_with_version(reports_path, docker_image_name) packages_path = os.path.join(temp_path, "packages") @@ -198,4 +205,4 @@ def run_stress_test(docker_image_name): if __name__ == "__main__": - run_stress_test("clickhouse/stress-test") + run_stress_test(f"{DOCKER_REPO}/clickhouse/stress-test") From 7b2280ddff282405dd5e7d7cc2462cdb5e298a10 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 02:02:24 -0400 Subject: [PATCH 167/213] run after building latest image --- .github/workflows/pull_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index ddbb87e42d0b..e523c91be13f 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -4,7 +4,6 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - on: # yamllint disable-line rule:truthy push: # pull_request: From ea330cb5641d8fc22b135d581ad00a56397a9d82 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 13:42:34 -0400 Subject: [PATCH 168/213] checking for build clanfTidy on new ibm vm --- .github/workflows/pull_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 1d1247ba01ee..39d8ed290348 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, test-only] + runs-on: [self-hosted, ibm-vm] steps: - name: test run: env @@ -40,7 +40,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, builder] + runs-on: [self-hosted, ibm-vm] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -57,7 +57,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, test-only] + runs-on: [self-hosted, ibm-vm] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -84,7 +84,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json BuilderBinClangTidy: needs: [DockerHubPush] - runs-on: [self-hosted, builder] + runs-on: [self-hosted, ibm-vm] steps: - name: Set envs run: | From 32bf0c4e5b3d87b9a5521638da90b1ceb172cad6 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 16:22:23 -0400 Subject: [PATCH 169/213] check value DOCKER_REPO --- .github/workflows/docker_server.yaml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/docker_server.yaml diff --git a/.github/workflows/docker_server.yaml b/.github/workflows/docker_server.yaml new file mode 100644 index 000000000000..fff202176b16 --- /dev/null +++ b/.github/workflows/docker_server.yaml @@ -0,0 +1,41 @@ +name: checkdocker + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## +jobs: + DockerServerImages: + runs-on: [self-hosted, build] + steps: + - name: Check value + run: | + DOCKER_REPO="${DOCKER_REPO:-docker.io}" + echo "DOCKER_REPO value: $DOCKER_REPO" + + + + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # DOCKER_REPO="${DOCKER_REPO:-docker.io}" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" \ No newline at end of file From 589054ce547ce69df4c6dda0cf47660ee19fc6a7 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 16:43:49 -0400 Subject: [PATCH 170/213] check value DOCKER_REPO --- .github/workflows/docker_server.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_server.yaml b/.github/workflows/docker_server.yaml index fff202176b16..eedc6e9ca656 100644 --- a/.github/workflows/docker_server.yaml +++ b/.github/workflows/docker_server.yaml @@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy ########################################################################################## jobs: DockerServerImages: - runs-on: [self-hosted, build] + runs-on: [self-hosted, docker] steps: - name: Check value run: | From 4a09b940e67c954ef2d5329b1f33e21fb6eaf633 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 23:05:09 -0400 Subject: [PATCH 171/213] checking for docker server image --- .github/workflows/pull_tests.yml | 1194 ++++++++++++++++++++++++++---- 1 file changed, 1070 insertions(+), 124 deletions(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 39d8ed290348..fce847a7328e 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -1,128 +1,128 @@ -name: PullclangCI +# name: PullclangCI -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 -on: # yamllint disable-line rule:truthy - push: - # pull_request: - # types: - # - synchronize - # - reopened - # - opened - # branches: - # - master - # paths-ignore: - # - 'CHANGELOG.md' - # - 'README.md' - # - 'SECURITY.md' - # - 'docker/docs/**' - # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' -jobs: - DockerHubPushAarch64: - runs-on: [self-hosted, ibm-vm] - steps: - - name: test - run: env - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - runs-on: [self-hosted, ibm-vm] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, ibm-vm] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json - BuilderBinClangTidy: - needs: [DockerHubPush] - runs-on: [self-hosted, ibm-vm] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_tidy - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# jobs: +# DockerHubPushAarch64: +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: test +# run: env +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json +# DockerHubPushAmd64: +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix amd64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json +# DockerHubPush: +# needs: [DockerHubPushAmd64, DockerHubPushAarch64] +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed aarch64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }} +# - name: Download changed amd64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }} +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/changed_images.json +# BuilderBinClangTidy: +# needs: [DockerHubPush] +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderBinDarwin: # needs: [DockerHubPush] # runs-on: [self-hosted, builder] @@ -304,4 +304,950 @@ jobs: # - name: Mark Commit Release Ready # run: | # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 mark_release_ready.py \ No newline at end of file + # python3 mark_release_ready.py + + +name: docker-check + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## +jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done + DockerHubPushAarch64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json +# StyleCheck: +# needs: DockerHubPush +# runs-on: [self-hosted, style-checker] +# # We need additional `&& ! cancelled()` to have the job being able to cancel +# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{ runner.temp }}/style_check +# ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/fasttest +# REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.TEMP_PATH }} +# - name: Fast Test +# run: | +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ## CompatibilityCheckX86: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckX86 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # CompatibilityCheckAarch64: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckAarch64 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ######################################################################################### +# # #################################### ORDINARY BUILDS #################################### +# # ######################################################################################### + BuilderDebRelease: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted, ibm-vm] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted, ibm-vm] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_asan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebUBsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_ubsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebTsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_tsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebMsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_msan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebDebug: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_debug +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ########################################################################################## +# # ##################################### SPECIAL BUILDS ##################################### +# # ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, ibm-vm] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + DOCKER_REPO="${DOCKER_REPO:-docker.io}" + python3 docker_server.py --release-type head --no-push \ + --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type head --no-push \ + --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" From a333e12ea103ad1b4a17f29932faaf229a5a5513 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 23:19:37 -0400 Subject: [PATCH 172/213] checking for docker server image --- .github/workflows/pull_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index fce847a7328e..201328884ea5 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -313,6 +313,7 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + on: # yamllint disable-line rule:truthy push: # pull_request: From 21b73dbf69a955025b32002961d584bd1b023029 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 19 Jun 2023 23:28:38 -0400 Subject: [PATCH 173/213] checking for docker server image --- .github/workflows/pull_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 201328884ea5..fce847a7328e 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -313,7 +313,6 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - on: # yamllint disable-line rule:truthy push: # pull_request: From 453a8260738201d13e6610a1fe51ae0cff2c87f5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 01:32:13 -0400 Subject: [PATCH 174/213] cehck for bianry image --- .github/workflows/binary_image.yml | 84 ++++++++++++++++++++++++++++++ docker/packager/binary/Dockerfile | 4 ++ docker/packager/binary/build.sh | 4 +- programs/diagnostics/Makefile | 2 +- 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/binary_image.yml diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml new file mode 100644 index 000000000000..62045f47c0c2 --- /dev/null +++ b/.github/workflows/binary_image.yml @@ -0,0 +1,84 @@ +name: binary-image + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, ibm-vm] + steps: + - name: test + run: env + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, ibm-vm] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, ibm-vm] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json \ No newline at end of file diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index ae8309953905..be81bb214d0f 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -111,5 +111,9 @@ COPY --from=cctools /cctools /cctools RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir +ARG GITHUB_REPOSITORY + +ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY + COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index c0803c741475..45298a68b239 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -x -e +export github_repository=${GITHUB_REPOSITORY:-ClickHouse/ClickHouse} + exec &> >(ts) ccache_status () { @@ -40,7 +42,7 @@ if [ -n "$MAKE_DEB" ]; then ( cd /build/programs/diagnostics make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build + GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" GITHUB_REPOSITORY="$github_repository" build mv clickhouse-diagnostics .. ) else diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 2e85002b871d..1c5372de07a3 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -5,7 +5,7 @@ BUILD_DIR=dist TIMESTAMP := $(shell date +%Y%m%d-%H%M) COMMIT := $(shell git rev-parse --short HEAD) -MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics +MODULE := github.com/${GITHUB_REPOSITORY}/programs/diagnostics VERSION := v.dev-${TIMESTAMP} DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" From ebcd4de6880db89abcdd4be41613bbb1ca99003b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 01:35:05 -0400 Subject: [PATCH 175/213] cehck for bianry image --- .github/workflows/binary_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 62045f47c0c2..51e40e922996 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, ibm-vm] + runs-on: [self-hosted, builder] steps: - name: test run: env @@ -40,7 +40,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, ibm-vm] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -57,7 +57,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, ibm-vm] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From 0ee318f6750c50b755bdab432ba8757fa2c64ef8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 02:09:08 -0400 Subject: [PATCH 176/213] check for builderdebRelease --- .github/workflows/pull_tests.yml | 138 +++++++++++++++---------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index fce847a7328e..c3b4a6f17e2e 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -556,7 +556,7 @@ jobs: BuilderDebRelease: # needs: [DockerHubPush, FastTest, StyleCheck] needs: [DockerHubPush] - runs-on: [self-hosted, ibm-vm] + runs-on: [self-hosted, builder] steps: - name: Set envs run: | @@ -637,49 +637,49 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - # needs: [DockerHubPush, FastTest, StyleCheck] - needs: [DockerHubPush] - runs-on: [self-hosted, ibm-vm] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAarch64: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush] + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderDebAsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -1226,28 +1226,28 @@ jobs: # ############################################################################################ # ##################################### Docker images ####################################### # ############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, ibm-vm] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - DOCKER_REPO="${DOCKER_REPO:-docker.io}" - python3 docker_server.py --release-type head --no-push \ - --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # DockerServerImages: + # needs: + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # DOCKER_REPO="${DOCKER_REPO:-docker.io}" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" From eeaf5126e732a9e999872712d83b074f14bddf66 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 09:53:58 -0400 Subject: [PATCH 177/213] check for binary image --- .github/workflows/pull_test.yaml | 7308 +++++++++++++++--------------- .github/workflows/pull_tests.yml | 2194 ++++----- docker/packager/binary/build.sh | 4 +- 3 files changed, 4753 insertions(+), 4753 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index e523c91be13f..33fcf42f0c93 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -1,294 +1,335 @@ -name: PullTestCI +# name: PullTestCI -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 -on: # yamllint disable-line rule:truthy - push: - # pull_request: - # types: - # - synchronize - # - reopened - # - opened - # branches: - # - master - # paths-ignore: - # - 'CHANGELOG.md' - # - 'README.md' - # - 'SECURITY.md' - # - 'docker/docs/**' - # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' -########################################################################################## -##################################### SMALL CHECKS ####################################### -########################################################################################## -jobs: - # CheckLabels: - # runs-on: [self-hosted, style-checker] - # # Run the first check always, even if the CI is cancelled - # if: ${{ always() }} - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Labels check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 run_check.py - # PythonUnitTests: - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Python unit tests - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # echo "Testing the main ci directory" - # python3 -m unittest discover -s . -p '*_test.py' - # for dir in *_lambda/; do - # echo "Testing $dir" - # python3 -m unittest discover -s "$dir" -p '*_test.py' - # done - DockerHubPushAarch64: - # needs: CheckLabels - runs-on: [self-hosted, test-only] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - # needs: CheckLabels - runs-on: [self-hosted, test-only] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, builder] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: echo "hi" - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{ runner.temp }}/style_check - # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/fasttest - # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # EOF - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.TEMP_PATH }} - # - name: Fast Test - # run: | - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -## CompatibilityCheckX86: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, style-checker] +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# ########################################################################################## +# ##################################### SMALL CHECKS ####################################### +# ########################################################################################## +# jobs: +# # CheckLabels: +# # runs-on: [self-hosted, style-checker] +# # # Run the first check always, even if the CI is cancelled +# # if: ${{ always() }} +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Labels check +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 run_check.py +# # PythonUnitTests: +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Python unit tests +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # echo "Testing the main ci directory" +# # python3 -m unittest discover -s . -p '*_test.py' +# # for dir in *_lambda/; do +# # echo "Testing $dir" +# # python3 -m unittest discover -s "$dir" -p '*_test.py' +# # done +# DockerHubPushAarch64: +# # needs: CheckLabels +# runs-on: [self-hosted, test-only] # steps: -# - name: Set envs +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check # run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/compatibility_check -# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# REPORTS_PATH=${{runner.temp}}/reports_dir -# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json +# DockerHubPushAmd64: +# # needs: CheckLabels +# runs-on: [self-hosted, test-only] +# steps: # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Download json reports +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix amd64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json +# DockerHubPush: +# needs: [DockerHubPushAmd64, DockerHubPushAarch64] +# runs-on: [self-hosted, builder] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed aarch64 images # uses: actions/download-artifact@v3 # with: -# path: ${{ env.REPORTS_PATH }} -# - name: CompatibilityCheckX86 -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions -# - name: Cleanup -# if: always() +# name: changed_images_aarch64 +# path: ${{ runner.temp }} +# - name: Download changed amd64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }} +# - name: Images check # run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# CompatibilityCheckAarch64: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, style-checker] +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/changed_images.json +# StyleCheck: +# needs: DockerHubPush +# runs-on: [self-hosted, builder] +# # We need additional `&& ! cancelled()` to have the job being able to cancel +# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# steps: +# - name: Set envs +# run: echo "hi" +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{ runner.temp }}/style_check +# # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/fasttest +# # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.TEMP_PATH }} +# # - name: Fast Test +# # run: | +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ## CompatibilityCheckX86: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckX86 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # CompatibilityCheckAarch64: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckAarch64 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ######################################################################################### +# # #################################### ORDINARY BUILDS #################################### +# # ######################################################################################### +# BuilderDebRelease: +# needs: [DockerHubPush, FastTest] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/compatibility_check -# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_release # EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: CompatibilityCheckAarch64 +# fetch-depth: 0 # for performance artifact +# submodules: true +# - name: Build # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ######################################################################################### -# #################################### ORDINARY BUILDS #################################### -# ######################################################################################### - BuilderDebRelease: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# BuilderBinRelease: +# # BuilderBinRelease: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_release +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAarch64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -299,18 +340,19 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_release +# BUILD_NAME=package_aarch64 # EOF # - name: Download changed images # uses: actions/download-artifact@v3 # with: # name: changed_images -# path: ${{ env.IMAGES_PATH }} +# path: ${{ runner.temp }}/images_path # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true # submodules: true +# fetch-depth: 0 # for performance artifact # - name: Build # run: | # sudo rm -fr "$TEMP_PATH" @@ -329,264 +371,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_asan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebUBsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_ubsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebTsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_tsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebMsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_msan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebDebug: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_debug - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Apply sparse checkout for contrib # in order to check that it doesn't break build - run: | - rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' - git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' - "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' - du -hs "$GITHUB_WORKSPACE/contrib" ||: - find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ########################################################################################## -# ##################################### SPECIAL BUILDS ##################################### -# ########################################################################################## -# BuilderBinClangTidy: +# BuilderDebAsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -597,7 +382,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_tidy +# BUILD_NAME=package_asan # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -627,48 +412,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwin: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAarch64: +# BuilderDebUBsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -679,7 +423,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_aarch64 +# BUILD_NAME=package_ubsan # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -709,7 +453,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinFreeBSD: +# BuilderDebTsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -720,7 +464,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_freebsd +# BUILD_NAME=package_tsan # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -750,48 +494,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinPPC64: +# BuilderDebMsan: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -802,7 +505,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_ppc64le +# BUILD_NAME=package_msan # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -832,7 +535,7 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAmd64Compat: +# BuilderDebDebug: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -843,7 +546,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_amd64_compat +# BUILD_NAME=package_debug # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -855,6 +558,13 @@ jobs: # with: # clear-repository: true # submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: # - name: Build # run: | # sudo rm -fr "$TEMP_PATH" @@ -873,7 +583,51 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAarch64V80Compat: +# # ########################################################################################## +# # ##################################### SPECIAL BUILDS ##################################### +# # ########################################################################################## +# # BuilderBinClangTidy: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_tidy +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -884,7 +638,7 @@ jobs: # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_aarch64_v80compat +# BUILD_NAME=binary_darwin # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -914,138 +668,454 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ############################################################################################ -# ##################################### Docker images ####################################### -# ############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# ############################################################################################ -# ##################################### BUILD REPORTER ####################################### -# ############################################################################################ -# BuilderReport: -# needs: -# - BuilderBinRelease -# - BuilderDebAarch64 -# - BuilderDebAsan -# - BuilderDebDebug -# - BuilderDebMsan -# - BuilderDebRelease -# - BuilderDebTsan -# - BuilderDebUBsan -# runs-on: [self-hosted, style-checker] -# if: ${{ success() || failure() }} +# # BuilderBinAarch64: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinFreeBSD: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_freebsd +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# CHECK_NAME=ClickHouse build check -# REPORTS_PATH=${{runner.temp}}/reports_dir -# TEMP_PATH=${{runner.temp}}/report_check -# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 # EOF -# - name: Download json reports +# - name: Download changed images # uses: actions/download-artifact@v3 # with: -# path: ${{ env.REPORTS_PATH }} +# name: changed_images +# path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Report Builder +# submodules: true +# - name: Build # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" -# cat > "$NEEDS_DATA_PATH" << 'EOF' -# ${{ toJSON(needs) }} -# EOF -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 build_report_check.py "$CHECK_NAME" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# BuilderSpecialReport: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinPPC64: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_ppc64le +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinAmd64Compat: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_amd64_compat +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinAarch64V80Compat: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_aarch64_v80compat +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ############################################################################################ +# # ##################################### Docker images ####################################### +# # ############################################################################################ +# DockerServerImages: # needs: -# - BuilderBinAarch64 -# - BuilderBinDarwin -# - BuilderBinDarwinAarch64 -# - BuilderBinFreeBSD -# - BuilderBinPPC64 -# - BuilderBinAmd64Compat -# - BuilderBinAarch64V80Compat -# - BuilderBinClangTidy +# - BuilderDebRelease +# - BuilderDebAarch64 # runs-on: [self-hosted, style-checker] -# if: ${{ success() || failure() }} # steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/report_check -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=ClickHouse special build check -# NEEDS_DATA_PATH=${{runner.temp}}/needs.json -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Report Builder +# fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself +# - name: Check docker clickhouse/clickhouse-server building # run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cat > "$NEEDS_DATA_PATH" << 'EOF' -# ${{ toJSON(needs) }} -# EOF # cd "$GITHUB_WORKSPACE/tests/ci" -# python3 build_report_check.py "$CHECK_NAME" +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-server --image-path docker/server +# python3 docker_server.py --release-type head --no-push \ +# --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################ -# #################################### INSTALL PACKAGES ###################################### -# ############################################################################################ -# InstallPackagesTestRelease: +# # ############################################################################################ +# # ##################################### BUILD REPORTER ####################################### +# # ############################################################################################ +# # BuilderReport: +# # needs: +# # - BuilderBinRelease +# # - BuilderDebAarch64 +# # - BuilderDebAsan +# # - BuilderDebDebug +# # - BuilderDebMsan +# # - BuilderDebRelease +# # - BuilderDebTsan +# # - BuilderDebUBsan +# # runs-on: [self-hosted, style-checker] +# # if: ${{ success() || failure() }} +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # CHECK_NAME=ClickHouse build check +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # TEMP_PATH=${{runner.temp}}/report_check +# # NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Report Builder +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cat > "$NEEDS_DATA_PATH" << 'EOF' +# # ${{ toJSON(needs) }} +# # EOF +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 build_report_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # BuilderSpecialReport: +# # needs: +# # - BuilderBinAarch64 +# # - BuilderBinDarwin +# # - BuilderBinDarwinAarch64 +# # - BuilderBinFreeBSD +# # - BuilderBinPPC64 +# # - BuilderBinAmd64Compat +# # - BuilderBinAarch64V80Compat +# # - BuilderBinClangTidy +# # runs-on: [self-hosted, style-checker] +# # if: ${{ success() || failure() }} +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/report_check +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=ClickHouse special build check +# # NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Report Builder +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cat > "$NEEDS_DATA_PATH" << 'EOF' +# # ${{ toJSON(needs) }} +# # EOF +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 build_report_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################ +# # #################################### INSTALL PACKAGES ###################################### +# # ############################################################################################ +# # InstallPackagesTestRelease: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/test_install +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Install packages (amd64) +# # REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Test packages installation +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 install_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # InstallPackagesTestAarch64: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, style-checker-aarch64] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/test_install +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Install packages (arm64) +# # REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Test packages installation +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 install_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################## +# # ########################### FUNCTIONAl STATELESS TESTS ####################################### +# # ############################################################################################## +# FunctionalStatelessTestRelease: # needs: [BuilderDebRelease] -# runs-on: [self-hosted, style-checker] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/test_install +# TEMP_PATH=${{runner.temp}}/stateless_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Install packages (amd64) -# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# CHECK_NAME=Stateless tests (release) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1055,30 +1125,711 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Test packages installation +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 install_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# InstallPackagesTestAarch64: +# # FunctionalStatelessTestReleaseDatabaseReplicated0: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseDatabaseReplicated1: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseDatabaseReplicated2: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseDatabaseReplicated3: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseWideParts: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, wide parts enabled) +# # REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# # KILL_TIMEOUT=10800 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseAnalyzer: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_analyzer +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, analyzer) +# # REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# # KILL_TIMEOUT=10800 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseS3_0: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=2 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestReleaseS3_1: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (release, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=2 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug0: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug1: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug2: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug3: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug4: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Debug5: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=5 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Tsan0: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Tsan1: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Tsan2: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Tsan3: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestS3Tsan4: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan, s3 storage) +# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAarch64: # needs: [BuilderDebAarch64] -# runs-on: [self-hosted, style-checker-aarch64] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/test_install +# TEMP_PATH=${{runner.temp}}/stateless_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Install packages (arm64) -# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# CHECK_NAME=Stateless tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1088,67 +1839,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Test packages installation +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 install_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ########################### FUNCTIONAl STATELESS TESTS ####################################### -# ############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseDatabaseReplicated0: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# TEMP_PATH=${{runner.temp}}/stateless_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=0 # RUN_BY_HASH_TOTAL=4 @@ -1174,17 +1888,17 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseDatabaseReplicated1: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# TEMP_PATH=${{runner.temp}}/stateless_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=1 # RUN_BY_HASH_TOTAL=4 @@ -1210,20 +1924,92 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseDatabaseReplicated2: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# # FunctionalStatelessTestAsan2: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (asan) +# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestAsan3: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (asan) +# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# TEMP_PATH=${{runner.temp}}/stateless_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1246,20 +2032,20 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseDatabaseReplicated3: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# TEMP_PATH=${{runner.temp}}/stateless_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1282,18 +2068,20 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseWideParts: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# TEMP_PATH=${{runner.temp}}/stateless_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, wide parts enabled) -# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse # KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1316,18 +2104,92 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseAnalyzer: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# # FunctionalStatelessTestTsan3: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan) +# # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestTsan4: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (tsan) +# # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan0: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# TEMP_PATH=${{runner.temp}}/stateless_ubsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, analyzer) -# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse # KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1350,20 +2212,56 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseS3_0: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# # FunctionalStatelessTestUBsan1: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (ubsan) +# # REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=2 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# TEMP_PATH=${{runner.temp}}/stateless_memory # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=2 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1386,20 +2284,20 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestReleaseS3_1: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestMsan1: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# TEMP_PATH=${{runner.temp}}/stateless_memory # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=2 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1422,19 +2320,19 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug0: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateless_memory # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_NUM=2 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -1458,20 +2356,128 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug1: +# # FunctionalStatelessTestMsan3: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_memory +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (msan) +# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestMsan4: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_memory +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (msan) +# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestMsan5: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_memory +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (msan) +# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=5 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug0: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateless_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1494,20 +2500,20 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug2: +# FunctionalStatelessTestDebug1: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateless_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1530,20 +2536,20 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug3: +# FunctionalStatelessTestDebug2: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateless_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1566,20 +2572,172 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug4: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# # FunctionalStatelessTestDebug3: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug) +# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestDebug4: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (debug) +# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=5 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestFlakyCheck: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests flaky check (asan) +# # REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # TestsBugfixCheck: +# # needs: [CheckLabels, StyleCheck] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=tests bugfix validate check +# # KILL_TIMEOUT=3600 +# # REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Bugfix test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" + +# # TEMP_PATH="${TEMP_PATH}/integration" \ +# # REPORTS_PATH="${REPORTS_PATH}/integration" \ +# # python3 integration_test_check.py "Integration $CHECK_NAME" \ +# # --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# # TEMP_PATH="${TEMP_PATH}/stateless" \ +# # REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# # python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# # --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# # python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################## +# # ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# # ############################################################################################## +# FunctionalStatefulTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateful_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=6 +# CHECK_NAME=Stateful tests (release) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1602,20 +2760,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Debug5: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] +# FunctionalStatefulTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# TEMP_PATH=${{runner.temp}}/stateful_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=5 -# RUN_BY_HASH_TOTAL=6 +# CHECK_NAME=Stateful tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1638,20 +2794,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Tsan0: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] +# FunctionalStatefulTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# TEMP_PATH=${{runner.temp}}/stateful_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=5 +# CHECK_NAME=Stateful tests (asan) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1674,2384 +2828,18 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Tsan1: +# FunctionalStatefulTestTsan: # needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Tsan2: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Tsan3: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestS3Tsan4: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan, s3 storage) -# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestAsan2: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (asan) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=4 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestAsan3: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (asan) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=4 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestTsan3: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan) -# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestTsan4: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan) -# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan0: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestUBsan1: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_ubsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (ubsan) -# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=2 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan3: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan4: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan5: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=5 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestDebug3: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestDebug4: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=5 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestFlakyCheck: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests flaky check (asan) -# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# TestsBugfixCheck: -# needs: [CheckLabels, StyleCheck] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/tests_bugfix_check -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=tests bugfix validate check -# KILL_TIMEOUT=3600 -# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Bugfix test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" - -# TEMP_PATH="${TEMP_PATH}/integration" \ -# REPORTS_PATH="${REPORTS_PATH}/integration" \ -# python3 integration_test_check.py "Integration $CHECK_NAME" \ -# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - -# TEMP_PATH="${TEMP_PATH}/stateless" \ -# REPORTS_PATH="${REPORTS_PATH}/stateless" \ -# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ -# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - -# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ############################ FUNCTIONAl STATEFUL TESTS ####################################### -# ############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# # Parallel replicas -# FunctionalStatefulTestDebugParallelReplicas: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (debug, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestUBsanParallelReplicas: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_ubsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestMsanParallelReplicas: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_msan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (msan, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestTsanParallelReplicas: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestAsanParallelReplicas: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (asan, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestReleaseParallelReplicas: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_release -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (release, ParallelReplicas) -# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse -# KILL_TIMEOUT=3600 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Functional test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ######################################### STRESS TESTS ####################################### -# ############################################################################################## - StressTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (asan) - REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestTsan: - needs: [BuilderDebTsan] - # func testers have 16 cores + 128 GB memory - # while stress testers have 36 cores + 72 memory - # It would be better to have something like 32 + 128, - # but such servers almost unavailable as spot instances. - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (tsan) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (msan) - REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_undefined - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (ubsan) - REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (debug) - REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ######################################### UPGRADE CHECK ###################################### -# ############################################################################################## -# UpgradeCheckAsan: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (asan) -# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Upgrade check -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# UpgradeCheckTsan: -# needs: [BuilderDebTsan] -# # same as for stress test with tsan -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_thread -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (tsan) -# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Upgrade check -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# UpgradeCheckMsan: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_memory -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (msan) -# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Upgrade check -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# UpgradeCheckDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/upgrade_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Upgrade check (debug) -# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Upgrade check -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 upgrade_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ##################################### AST FUZZERS ############################################ -# ############################################################################################## -# ASTFuzzerTestAsan: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (asan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestTsan: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (tsan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestUBSan: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (ubsan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestMSan: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (msan) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ASTFuzzerTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=AST fuzzer (debug) -# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Fuzzer -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 ast_fuzzer_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# ############################################################################################# -# ############################# INTEGRATION TESTS ############################################# -# ############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan3: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan4: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan5: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# RUN_BY_HASH_NUM=5 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan4: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, stress-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=4 -# RUN_BY_HASH_TOTAL=6 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Integration test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan5: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, stress-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/stateful_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# RUN_BY_HASH_NUM=5 -# RUN_BY_HASH_TOTAL=6 +# CHECK_NAME=Stateful tests (tsan) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4061,102 +2849,31 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease2: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# FunctionalStatefulTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release +# TEMP_PATH=${{runner.temp}}/stateful_msan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Stateful tests (msan) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4166,32 +2883,31 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease3: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# FunctionalStatefulTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release +# TEMP_PATH=${{runner.temp}}/stateful_ubsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Stateful tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4201,30 +2917,31 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsFlakyCheck: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, stress-tester] +# FunctionalStatefulTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# TEMP_PATH=${{runner.temp}}/stateful_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests flaky check (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# CHECK_NAME=Stateful tests (debug) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4234,66 +2951,244 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Functional test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################# -# #################################### UNIT TESTS ############################################# -# ############################################################################################# -# UnitTestsAsan: +# # # Parallel replicas +# # FunctionalStatefulTestDebugParallelReplicas: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatefulTestUBsanParallelReplicas: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatefulTestMsanParallelReplicas: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_msan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatefulTestTsanParallelReplicas: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatefulTestAsanParallelReplicas: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # FunctionalStatefulTestReleaseParallelReplicas: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateful_release +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateful tests (release, ParallelReplicas) +# # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# # KILL_TIMEOUT=3600 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################## +# # ######################################### STRESS TESTS ####################################### +# # ############################################################################################## +# StressTestAsan: # needs: [BuilderDebAsan] -# runs-on: [self-hosted, fuzzer-unit-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/unit_tests_asan -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Unit tests (asan) -# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v3 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Unit test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 unit_tests_check.py "$CHECK_NAME" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" -# UnitTestsReleaseClang: -# needs: [BuilderBinRelease] -# runs-on: [self-hosted, fuzzer-unit-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# TEMP_PATH=${{runner.temp}}/stress_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Unit tests (release-clang) -# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# CHECK_NAME=Stress test (asan) +# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4303,30 +3198,34 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Unit test +# - name: Stress test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 unit_tests_check.py "$CHECK_NAME" +# python3 stress_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UnitTestsTsan: +# StressTestTsan: # needs: [BuilderDebTsan] -# runs-on: [self-hosted, fuzzer-unit-tester] +# # func testers have 16 cores + 128 GB memory +# # while stress testers have 36 cores + 72 memory +# # It would be better to have something like 32 + 128, +# # but such servers almost unavailable as spot instances. +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# TEMP_PATH=${{runner.temp}}/stress_thread # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Unit tests (tsan) -# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# CHECK_NAME=Stress test (tsan) +# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4336,30 +3235,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Unit test +# - name: Stress test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 unit_tests_check.py "$CHECK_NAME" +# python3 stress_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UnitTestsMsan: +# StressTestMsan: # needs: [BuilderDebMsan] -# runs-on: [self-hosted, fuzzer-unit-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# TEMP_PATH=${{runner.temp}}/stress_memory # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Unit tests (msan) -# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# CHECK_NAME=Stress test (msan) +# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4369,30 +3268,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Unit test +# - name: Stress test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 unit_tests_check.py "$CHECK_NAME" +# python3 stress_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# UnitTestsUBsan: +# StressTestUBsan: # needs: [BuilderDebUBsan] -# runs-on: [self-hosted, fuzzer-unit-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# TEMP_PATH=${{runner.temp}}/stress_undefined # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Unit tests (ubsan) -# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# CHECK_NAME=Stress test (ubsan) +# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4402,35 +3301,30 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Unit test +# - name: Stress test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 unit_tests_check.py "$CHECK_NAME" +# python3 stress_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################# -# #################################### PERFORMANCE TESTS ###################################### -# ############################################################################################# -# PerformanceComparisonX86-0: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# StressTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/stress_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Stress test (debug) +# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4440,32 +3334,339 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Stress test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 stress_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonX86-1: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# # ############################################################################################## +# # ######################################### UPGRADE CHECK ###################################### +# # ############################################################################################## +# # UpgradeCheckAsan: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/upgrade_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Upgrade check (asan) +# # REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Upgrade check +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 upgrade_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UpgradeCheckTsan: +# # needs: [BuilderDebTsan] +# # # same as for stress test with tsan +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/upgrade_thread +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Upgrade check (tsan) +# # REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Upgrade check +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 upgrade_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UpgradeCheckMsan: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/upgrade_memory +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Upgrade check (msan) +# # REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Upgrade check +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 upgrade_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UpgradeCheckDebug: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/upgrade_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Upgrade check (debug) +# # REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Upgrade check +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 upgrade_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################## +# # ##################################### AST FUZZERS ############################################ +# # ############################################################################################## +# # ASTFuzzerTestAsan: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=AST fuzzer (asan) +# # REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Fuzzer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 ast_fuzzer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ASTFuzzerTestTsan: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=AST fuzzer (tsan) +# # REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Fuzzer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 ast_fuzzer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ASTFuzzerTestUBSan: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=AST fuzzer (ubsan) +# # REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Fuzzer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 ast_fuzzer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ASTFuzzerTestMSan: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=AST fuzzer (msan) +# # REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Fuzzer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 ast_fuzzer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ASTFuzzerTestDebug: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=AST fuzzer (debug) +# # REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Fuzzer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 ast_fuzzer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################# +# # ############################# INTEGRATION TESTS ############################################# +# # ############################################################################################# +# IntegrationTestsAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4475,32 +3676,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonX86-2: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# IntegrationTestsAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4510,32 +3711,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonX86-3: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, stress-tester] +# IntegrationTestsAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4545,32 +3746,137 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonAarch-0: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, func-tester-aarch64] +# # IntegrationTestsAsan3: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (asan) +# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # IntegrationTestsAsan4: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (asan) +# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # IntegrationTestsAsan5: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (asan) +# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# # RUN_BY_HASH_NUM=5 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison Aarch64 -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse # RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4580,32 +3886,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonAarch-1: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, func-tester-aarch64] +# IntegrationTestsTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison Aarch64 -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse # RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4615,32 +3921,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonAarch-2: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, func-tester-aarch64] +# IntegrationTestsTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison Aarch64 -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse # RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4650,32 +3956,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# PerformanceComparisonAarch-3: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, func-tester-aarch64] +# IntegrationTestsTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/performance_comparison +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Performance Comparison Aarch64 -# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse # RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4685,33 +3991,102 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Performance Comparison +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 performance_comparison_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################## -# ###################################### SQLANCER FUZZERS ###################################### -# ############################################################################################## -# SQLancerTestRelease: +# # IntegrationTestsTsan4: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (tsan) +# # REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# # RUN_BY_HASH_NUM=4 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # IntegrationTestsTsan5: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (tsan) +# # REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# # RUN_BY_HASH_NUM=5 +# # RUN_BY_HASH_TOTAL=6 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease0: # needs: [BuilderDebRelease] -# runs-on: [self-hosted, fuzzer-unit-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/sqlancer_release +# TEMP_PATH=${{runner.temp}}/integration_tests_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=SQLancer (release) -# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4721,30 +4096,32 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: SQLancer +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 sqlancer_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# SQLancerTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, fuzzer-unit-tester] +# IntegrationTestsRelease1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# TEMP_PATH=${{runner.temp}}/integration_tests_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=SQLancer (debug) -# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4754,176 +4131,799 @@ jobs: # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: SQLancer +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 sqlancer_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# ############################################################################################# -# ###################################### JEPSEN TESTS ######################################### -# ############################################################################################# -# Jepsen: -# # This is special test NOT INCLUDED in FinishCheck -# # When it's skipped, all dependent tasks will be skipped too. -# # DO NOT add it there -# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') -# needs: [BuilderBinRelease] -# uses: ./.github/workflows/jepsen.yml -# FinishCheck: -# needs: -# - StyleCheck -# - DockerHubPush -# - DockerServerImages -# - CheckLabels -# - BuilderReport -# - BuilderSpecialReport -# - FastTest -# - FunctionalStatelessTestDebug0 -# - FunctionalStatelessTestDebug1 -# - FunctionalStatelessTestDebug2 -# - FunctionalStatelessTestDebug3 -# - FunctionalStatelessTestDebug4 -# - FunctionalStatelessTestRelease -# - FunctionalStatelessTestReleaseDatabaseReplicated0 -# - FunctionalStatelessTestReleaseDatabaseReplicated1 -# - FunctionalStatelessTestReleaseDatabaseReplicated2 -# - FunctionalStatelessTestReleaseDatabaseReplicated3 -# - FunctionalStatelessTestReleaseWideParts -# - FunctionalStatelessTestReleaseAnalyzer -# - FunctionalStatelessTestAarch64 -# - FunctionalStatelessTestAsan0 -# - FunctionalStatelessTestAsan1 -# - FunctionalStatelessTestAsan2 -# - FunctionalStatelessTestAsan3 -# - FunctionalStatelessTestTsan0 -# - FunctionalStatelessTestTsan1 -# - FunctionalStatelessTestTsan2 -# - FunctionalStatelessTestTsan3 -# - FunctionalStatelessTestTsan4 -# - FunctionalStatelessTestMsan0 -# - FunctionalStatelessTestMsan1 -# - FunctionalStatelessTestMsan2 -# - FunctionalStatelessTestMsan3 -# - FunctionalStatelessTestMsan4 -# - FunctionalStatelessTestMsan5 -# - FunctionalStatelessTestUBsan0 -# - FunctionalStatelessTestUBsan1 -# - FunctionalStatefulTestDebug -# - FunctionalStatefulTestRelease -# - FunctionalStatefulTestAarch64 -# - FunctionalStatefulTestAsan -# - FunctionalStatefulTestTsan -# - FunctionalStatefulTestMsan -# - FunctionalStatefulTestUBsan -# - FunctionalStatelessTestReleaseS3_0 -# - FunctionalStatelessTestReleaseS3_1 -# - FunctionalStatelessTestS3Debug0 -# - FunctionalStatelessTestS3Debug1 -# - FunctionalStatelessTestS3Debug2 -# - FunctionalStatelessTestS3Debug4 -# - FunctionalStatelessTestS3Debug5 -# - FunctionalStatelessTestS3Tsan0 -# - FunctionalStatelessTestS3Tsan1 -# - FunctionalStatelessTestS3Tsan2 -# - FunctionalStatelessTestS3Tsan4 -# - StressTestDebug -# - StressTestAsan -# - StressTestTsan -# - StressTestMsan -# - StressTestUBsan -# - ASTFuzzerTestDebug -# - ASTFuzzerTestAsan -# - ASTFuzzerTestTsan -# - ASTFuzzerTestMSan -# - ASTFuzzerTestUBSan -# - IntegrationTestsAsan0 -# - IntegrationTestsAsan1 -# - IntegrationTestsAsan2 -# - IntegrationTestsAsan3 -# - IntegrationTestsAsan4 -# - IntegrationTestsAsan5 -# - IntegrationTestsRelease0 -# - IntegrationTestsRelease1 -# - IntegrationTestsRelease2 -# - IntegrationTestsRelease3 -# - IntegrationTestsTsan0 -# - IntegrationTestsTsan1 -# - IntegrationTestsTsan2 -# - IntegrationTestsTsan3 -# - IntegrationTestsTsan4 -# - IntegrationTestsTsan5 -# - PerformanceComparisonX86-0 -# - PerformanceComparisonX86-1 -# - PerformanceComparisonX86-2 -# - PerformanceComparisonX86-3 -# - PerformanceComparisonAarch-0 -# - PerformanceComparisonAarch-1 -# - PerformanceComparisonAarch-2 -# - PerformanceComparisonAarch-3 -# - UnitTestsAsan -# - UnitTestsTsan -# - UnitTestsMsan -# - UnitTestsUBsan -# - UnitTestsReleaseClang -# - CompatibilityCheckX86 -# - CompatibilityCheckAarch64 -# - IntegrationTestsFlakyCheck -# - SQLancerTestRelease -# - SQLancerTestDebug -# runs-on: [self-hosted, style-checker] -# steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Finish label -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 finish_check.py -# python3 merge_pr.py --check-approved -# ############################################################################################## -# ########################### SQLLOGIC TEST ################################################### -# ############################################################################################## -# SQLLogicTestRelease: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, func-tester] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/sqllogic_debug -# REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Sqllogic test (release) -# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse -# KILL_TIMEOUT=10800 -# EOF -# - name: Download json reports -# uses: actions/download-artifact@v2 -# with: -# path: ${{ env.REPORTS_PATH }} -# - name: Clear repository -# run: | -# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# - name: Check out repository code -# uses: actions/checkout@v2 -# - name: Sqllogic test -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" -# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" \ No newline at end of file +# # IntegrationTestsRelease2: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_release +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (release) +# # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # IntegrationTestsRelease3: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_release +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests (release) +# # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # IntegrationTestsFlakyCheck: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Integration tests flaky check (asan) +# # REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Integration test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 integration_test_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################# +# # #################################### UNIT TESTS ############################################# +# # ############################################################################################# +# # UnitTestsAsan: +# # needs: [BuilderDebAsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/unit_tests_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Unit tests (asan) +# # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Unit test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 unit_tests_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UnitTestsReleaseClang: +# # needs: [BuilderBinRelease] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/unit_tests_asan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Unit tests (release-clang) +# # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Unit test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 unit_tests_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UnitTestsTsan: +# # needs: [BuilderDebTsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Unit tests (tsan) +# # REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Unit test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 unit_tests_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UnitTestsMsan: +# # needs: [BuilderDebMsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/unit_tests_msan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Unit tests (msan) +# # REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Unit test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 unit_tests_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # UnitTestsUBsan: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Unit tests (ubsan) +# # REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Unit test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 unit_tests_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################# +# # #################################### PERFORMANCE TESTS ###################################### +# # ############################################################################################# +# # PerformanceComparisonX86-0: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonX86-1: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonX86-2: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonX86-3: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, stress-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonAarch-0: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, func-tester-aarch64] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison Aarch64 +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=0 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonAarch-1: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, func-tester-aarch64] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison Aarch64 +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonAarch-2: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, func-tester-aarch64] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison Aarch64 +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=2 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # PerformanceComparisonAarch-3: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, func-tester-aarch64] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/performance_comparison +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Performance Comparison Aarch64 +# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# # RUN_BY_HASH_NUM=3 +# # RUN_BY_HASH_TOTAL=4 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Performance Comparison +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 performance_comparison_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################## +# # ###################################### SQLANCER FUZZERS ###################################### +# # ############################################################################################## +# # SQLancerTestRelease: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/sqlancer_release +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=SQLancer (release) +# # REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: SQLancer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 sqlancer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # SQLancerTestDebug: +# # needs: [BuilderDebDebug] +# # runs-on: [self-hosted, fuzzer-unit-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/sqlancer_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=SQLancer (debug) +# # REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: SQLancer +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 sqlancer_check.py "$CHECK_NAME" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ############################################################################################# +# # ###################################### JEPSEN TESTS ######################################### +# # ############################################################################################# +# # Jepsen: +# # # This is special test NOT INCLUDED in FinishCheck +# # # When it's skipped, all dependent tasks will be skipped too. +# # # DO NOT add it there +# # if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# # needs: [BuilderBinRelease] +# # uses: ./.github/workflows/jepsen.yml +# # FinishCheck: +# # needs: +# # - StyleCheck +# # - DockerHubPush +# # - DockerServerImages +# # - CheckLabels +# # - BuilderReport +# # - BuilderSpecialReport +# # - FastTest +# # - FunctionalStatelessTestDebug0 +# # - FunctionalStatelessTestDebug1 +# # - FunctionalStatelessTestDebug2 +# # - FunctionalStatelessTestDebug3 +# # - FunctionalStatelessTestDebug4 +# # - FunctionalStatelessTestRelease +# # - FunctionalStatelessTestReleaseDatabaseReplicated0 +# # - FunctionalStatelessTestReleaseDatabaseReplicated1 +# # - FunctionalStatelessTestReleaseDatabaseReplicated2 +# # - FunctionalStatelessTestReleaseDatabaseReplicated3 +# # - FunctionalStatelessTestReleaseWideParts +# # - FunctionalStatelessTestReleaseAnalyzer +# # - FunctionalStatelessTestAarch64 +# # - FunctionalStatelessTestAsan0 +# # - FunctionalStatelessTestAsan1 +# # - FunctionalStatelessTestAsan2 +# # - FunctionalStatelessTestAsan3 +# # - FunctionalStatelessTestTsan0 +# # - FunctionalStatelessTestTsan1 +# # - FunctionalStatelessTestTsan2 +# # - FunctionalStatelessTestTsan3 +# # - FunctionalStatelessTestTsan4 +# # - FunctionalStatelessTestMsan0 +# # - FunctionalStatelessTestMsan1 +# # - FunctionalStatelessTestMsan2 +# # - FunctionalStatelessTestMsan3 +# # - FunctionalStatelessTestMsan4 +# # - FunctionalStatelessTestMsan5 +# # - FunctionalStatelessTestUBsan0 +# # - FunctionalStatelessTestUBsan1 +# # - FunctionalStatefulTestDebug +# # - FunctionalStatefulTestRelease +# # - FunctionalStatefulTestAarch64 +# # - FunctionalStatefulTestAsan +# # - FunctionalStatefulTestTsan +# # - FunctionalStatefulTestMsan +# # - FunctionalStatefulTestUBsan +# # - FunctionalStatelessTestReleaseS3_0 +# # - FunctionalStatelessTestReleaseS3_1 +# # - FunctionalStatelessTestS3Debug0 +# # - FunctionalStatelessTestS3Debug1 +# # - FunctionalStatelessTestS3Debug2 +# # - FunctionalStatelessTestS3Debug4 +# # - FunctionalStatelessTestS3Debug5 +# # - FunctionalStatelessTestS3Tsan0 +# # - FunctionalStatelessTestS3Tsan1 +# # - FunctionalStatelessTestS3Tsan2 +# # - FunctionalStatelessTestS3Tsan4 +# # - StressTestDebug +# # - StressTestAsan +# # - StressTestTsan +# # - StressTestMsan +# # - StressTestUBsan +# # - ASTFuzzerTestDebug +# # - ASTFuzzerTestAsan +# # - ASTFuzzerTestTsan +# # - ASTFuzzerTestMSan +# # - ASTFuzzerTestUBSan +# # - IntegrationTestsAsan0 +# # - IntegrationTestsAsan1 +# # - IntegrationTestsAsan2 +# # - IntegrationTestsAsan3 +# # - IntegrationTestsAsan4 +# # - IntegrationTestsAsan5 +# # - IntegrationTestsRelease0 +# # - IntegrationTestsRelease1 +# # - IntegrationTestsRelease2 +# # - IntegrationTestsRelease3 +# # - IntegrationTestsTsan0 +# # - IntegrationTestsTsan1 +# # - IntegrationTestsTsan2 +# # - IntegrationTestsTsan3 +# # - IntegrationTestsTsan4 +# # - IntegrationTestsTsan5 +# # - PerformanceComparisonX86-0 +# # - PerformanceComparisonX86-1 +# # - PerformanceComparisonX86-2 +# # - PerformanceComparisonX86-3 +# # - PerformanceComparisonAarch-0 +# # - PerformanceComparisonAarch-1 +# # - PerformanceComparisonAarch-2 +# # - PerformanceComparisonAarch-3 +# # - UnitTestsAsan +# # - UnitTestsTsan +# # - UnitTestsMsan +# # - UnitTestsUBsan +# # - UnitTestsReleaseClang +# # - CompatibilityCheckX86 +# # - CompatibilityCheckAarch64 +# # - IntegrationTestsFlakyCheck +# # - SQLancerTestRelease +# # - SQLancerTestDebug +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Finish label +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 finish_check.py +# # python3 merge_pr.py --check-approved +# # ############################################################################################## +# # ########################### SQLLOGIC TEST ################################################### +# # ############################################################################################## +# # SQLLogicTestRelease: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/sqllogic_debug +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Sqllogic test (release) +# # REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# # KILL_TIMEOUT=10800 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v2 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Clear repository +# # run: | +# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# # - name: Check out repository code +# # uses: actions/checkout@v2 +# # - name: Sqllogic test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" \ No newline at end of file diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index c3b4a6f17e2e..0846e4cbe893 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -1,4 +1,313 @@ -# name: PullclangCI +# # name: PullclangCI + +# # env: +# # # Force the stdout and stderr streams to be unbuffered +# # PYTHONUNBUFFERED: 1 + +# # on: # yamllint disable-line rule:truthy +# # push: +# # # pull_request: +# # # types: +# # # - synchronize +# # # - reopened +# # # - opened +# # # branches: +# # # - master +# # # paths-ignore: +# # # - 'CHANGELOG.md' +# # # - 'README.md' +# # # - 'SECURITY.md' +# # # - 'docker/docs/**' +# # # - 'docs/**' +# # # - 'utils/check-style/aspell-ignore/**' +# # jobs: +# # DockerHubPushAarch64: +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: test +# # run: env +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Images check +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 docker_images_check.py --suffix aarch64 +# # - name: Upload images files to artifacts +# # uses: actions/upload-artifact@v3 +# # with: +# # name: changed_images_aarch64 +# # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json +# # DockerHubPushAmd64: +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Images check +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 docker_images_check.py --suffix amd64 +# # - name: Upload images files to artifacts +# # uses: actions/upload-artifact@v3 +# # with: +# # name: changed_images_amd64 +# # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json +# # DockerHubPush: +# # needs: [DockerHubPushAmd64, DockerHubPushAarch64] +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download changed aarch64 images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images_aarch64 +# # path: ${{ runner.temp }} +# # - name: Download changed amd64 images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images_amd64 +# # path: ${{ runner.temp }} +# # - name: Images check +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 +# # - name: Upload images files to artifacts +# # uses: actions/upload-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ runner.temp }}/changed_images.json +# # BuilderBinClangTidy: +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_tidy +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinDarwin: +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_darwin +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinDarwinAarch64: +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_darwin_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebRelease: +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_release +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # fetch-depth: 0 # for performance artifact +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # BuilderDebAarch64: +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, test-only] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ runner.temp }}/images_path +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # fetch-depth: 0 # for performance artifact +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # MarkReleaseReady: +# # needs: +# # - BuilderBinDarwin +# # - BuilderBinDarwinAarch64 +# # - BuilderDebRelease +# # - BuilderDebAarch64 +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Mark Commit Release Ready +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 mark_release_ready.py + + +# name: docker-check # env: # # Force the stdout and stderr streams to be unbuffered @@ -19,13 +328,44 @@ # # - 'SECURITY.md' # # - 'docker/docs/**' # # - 'docs/**' -# # - 'utils/check-style/aspell-ignore/**' +# # - 'utils/check-style/aspell-ignore/**' +# ########################################################################################## +# ##################################### SMALL CHECKS ####################################### +# ########################################################################################## # jobs: +# # CheckLabels: +# # runs-on: [self-hosted, style-checker] +# # # Run the first check always, even if the CI is cancelled +# # if: ${{ always() }} +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Labels check +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # python3 run_check.py +# # PythonUnitTests: +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Python unit tests +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # echo "Testing the main ci directory" +# # python3 -m unittest discover -s . -p '*_test.py' +# # for dir in *_lambda/; do +# # echo "Testing $dir" +# # python3 -m unittest discover -s "$dir" -p '*_test.py' +# # done # DockerHubPushAarch64: -# runs-on: [self-hosted, ibm-vm] +# # needs: CheckLabels +# runs-on: [self-hosted, style-checker] # steps: -# - name: test -# run: env # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: @@ -40,7 +380,8 @@ # name: changed_images_aarch64 # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # DockerHubPushAmd64: -# runs-on: [self-hosted, ibm-vm] +# # needs: CheckLabels +# runs-on: [self-hosted, style-checker] # steps: # - name: Check out repository code # uses: ClickHouse/checkout@v1 @@ -57,7 +398,7 @@ # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json # DockerHubPush: # needs: [DockerHubPushAmd64, DockerHubPushAarch64] -# runs-on: [self-hosted, ibm-vm] +# runs-on: [self-hosted, style-checker] # steps: # - name: Check out repository code # uses: ClickHouse/checkout@v1 @@ -82,9 +423,140 @@ # with: # name: changed_images # path: ${{ runner.temp }}/changed_images.json -# BuilderBinClangTidy: -# needs: [DockerHubPush] -# runs-on: [self-hosted, ibm-vm] +# # StyleCheck: +# # needs: DockerHubPush +# # runs-on: [self-hosted, style-checker] +# # # We need additional `&& ! cancelled()` to have the job being able to cancel +# # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{ runner.temp }}/style_check +# # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/fasttest +# # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.TEMP_PATH }} +# # - name: Fast Test +# # run: | +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ## CompatibilityCheckX86: +# # # needs: [BuilderDebRelease] +# # # runs-on: [self-hosted, style-checker] +# # # steps: +# # # - name: Set envs +# # # run: | +# # # cat >> "$GITHUB_ENV" << 'EOF' +# # # TEMP_PATH=${{runner.temp}}/compatibility_check +# # # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # # REPORTS_PATH=${{runner.temp}}/reports_dir +# # # EOF +# # # - name: Check out repository code +# # # uses: ClickHouse/checkout@v1 +# # # with: +# # # clear-repository: true +# # # - name: Download json reports +# # # uses: actions/download-artifact@v3 +# # # with: +# # # path: ${{ env.REPORTS_PATH }} +# # # - name: CompatibilityCheckX86 +# # # run: | +# # # sudo rm -fr "$TEMP_PATH" +# # # mkdir -p "$TEMP_PATH" +# # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# # # - name: Cleanup +# # # if: always() +# # # run: | +# # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # # sudo rm -fr "$TEMP_PATH" +# # # CompatibilityCheckAarch64: +# # # needs: [BuilderDebAarch64] +# # # runs-on: [self-hosted, style-checker] +# # # steps: +# # # - name: Set envs +# # # run: | +# # # cat >> "$GITHUB_ENV" << 'EOF' +# # # TEMP_PATH=${{runner.temp}}/compatibility_check +# # # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # # REPORTS_PATH=${{runner.temp}}/reports_dir +# # # EOF +# # # - name: Check out repository code +# # # uses: ClickHouse/checkout@v1 +# # # with: +# # # clear-repository: true +# # # - name: Download json reports +# # # uses: actions/download-artifact@v3 +# # # with: +# # # path: ${{ env.REPORTS_PATH }} +# # # - name: CompatibilityCheckAarch64 +# # # run: | +# # # sudo rm -fr "$TEMP_PATH" +# # # mkdir -p "$TEMP_PATH" +# # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# # # - name: Cleanup +# # # if: always() +# # # run: | +# # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # # sudo rm -fr "$TEMP_PATH" +# # # ######################################################################################### +# # # #################################### ORDINARY BUILDS #################################### +# # # ######################################################################################### +# BuilderDebRelease: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# needs: [DockerHubPush] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | @@ -93,7 +565,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_tidy +# BUILD_NAME=package_release # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -104,6 +576,7 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true +# fetch-depth: 0 # for performance artifact # submodules: true # - name: Build # run: | @@ -122,1132 +595,659 @@ # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinDarwin: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_darwin - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinDarwinAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_darwin_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderDebRelease: - # needs: [DockerHubPush] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_release - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ env.IMAGES_PATH }} - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # fetch-depth: 0 # for performance artifact - # submodules: true - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" - # BuilderDebAarch64: - # needs: [DockerHubPush] - # runs-on: [self-hosted, test-only] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # fetch-depth: 0 # for performance artifact - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # MarkReleaseReady: - # needs: - # - BuilderBinDarwin - # - BuilderBinDarwinAarch64 - # - BuilderDebRelease - # - BuilderDebAarch64 - # runs-on: [self-hosted, builder] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Mark Commit Release Ready - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 mark_release_ready.py - - -name: docker-check - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - -on: # yamllint disable-line rule:truthy - push: - # pull_request: - # types: - # - synchronize - # - reopened - # - opened - # branches: - # - master - # paths-ignore: - # - 'CHANGELOG.md' - # - 'README.md' - # - 'SECURITY.md' - # - 'docker/docs/**' - # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' -########################################################################################## -##################################### SMALL CHECKS ####################################### -########################################################################################## -jobs: - # CheckLabels: - # runs-on: [self-hosted, style-checker] - # # Run the first check always, even if the CI is cancelled - # if: ${{ always() }} - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Labels check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 run_check.py - # PythonUnitTests: - # runs-on: [self-hosted, style-checker] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Python unit tests - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # echo "Testing the main ci directory" - # python3 -m unittest discover -s . -p '*_test.py' - # for dir in *_lambda/; do - # echo "Testing $dir" - # python3 -m unittest discover -s "$dir" -p '*_test.py' - # done - DockerHubPushAarch64: - # needs: CheckLabels - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - # needs: CheckLabels - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json -# StyleCheck: -# needs: DockerHubPush -# runs-on: [self-hosted, style-checker] -# # We need additional `&& ! cancelled()` to have the job being able to cancel -# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{ runner.temp }}/style_check -# ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/fasttest -# REPO_COPY=${{runner.temp}}/fasttest/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# EOF -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.TEMP_PATH }} -# - name: Fast Test -# run: | -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ## CompatibilityCheckX86: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, style-checker] +# # BuilderBinRelease: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] # # steps: # # - name: Set envs # # run: | # # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/compatibility_check -# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_release # # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} # # - name: Check out repository code # # uses: ClickHouse/checkout@v1 # # with: # # clear-repository: true -# # - name: Download json reports +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebAarch64: +# # # needs: [DockerHubPush, FastTest, StyleCheck] +# # needs: [DockerHubPush] +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ runner.temp }}/images_path +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # fetch-depth: 0 # for performance artifact +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebAsan: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_asan +# # EOF +# # - name: Download changed images # # uses: actions/download-artifact@v3 # # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: CompatibilityCheckX86 +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build # # run: | # # sudo rm -fr "$TEMP_PATH" # # mkdir -p "$TEMP_PATH" # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json # # - name: Cleanup # # if: always() # # run: | # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebUBsan: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_ubsan +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | # # sudo rm -fr "$TEMP_PATH" -# # CompatibilityCheckAarch64: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, style-checker] +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebTsan: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] # # steps: # # - name: Set envs # # run: | # # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/compatibility_check -# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_tsan # # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} # # - name: Check out repository code # # uses: ClickHouse/checkout@v1 # # with: # # clear-repository: true -# # - name: Download json reports +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebMsan: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_msan +# # EOF +# # - name: Download changed images # # uses: actions/download-artifact@v3 # # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: CompatibilityCheckAarch64 +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build # # run: | # # sudo rm -fr "$TEMP_PATH" # # mkdir -p "$TEMP_PATH" # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json # # - name: Cleanup # # if: always() # # run: | # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderDebDebug: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=package_debug +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# # run: | +# # rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# # git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# # "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# # du -hs "$GITHUB_WORKSPACE/contrib" ||: +# # find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# # - name: Build +# # run: | # # sudo rm -fr "$TEMP_PATH" -# # ######################################################################################### -# # #################################### ORDINARY BUILDS #################################### -# # ######################################################################################### - BuilderDebRelease: - # needs: [DockerHubPush, FastTest, StyleCheck] - needs: [DockerHubPush] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# BuilderBinRelease: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_release -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderDebAarch64: - # # needs: [DockerHubPush, FastTest, StyleCheck] - # needs: [DockerHubPush] - # runs-on: [self-hosted, ibm-vm] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=package_aarch64 - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # submodules: true - # fetch-depth: 0 # for performance artifact - # - name: Build - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebAsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_asan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebUBsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_ubsan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebTsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_tsan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebMsan: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_msan -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebDebug: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_debug -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build -# run: | -# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' -# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' -# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' -# du -hs "$GITHUB_WORKSPACE/contrib" ||: -# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # ########################################################################################## -# # ##################################### SPECIAL BUILDS ##################################### -# # ########################################################################################## -# BuilderBinClangTidy: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_tidy -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwin: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAarch64: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_aarch64 -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinFreeBSD: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_freebsd -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwinAarch64: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin_aarch64 -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinPPC64: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_ppc64le -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAmd64Compat: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_amd64_compat -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinAarch64V80Compat: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_aarch64_v80compat -# EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# submodules: true -# - name: Build -# run: | -# sudo rm -fr "$TEMP_PATH" -# mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# - name: Cleanup -# if: always() -# run: | -# docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ############################################################################################ -# ##################################### Docker images ####################################### -# ############################################################################################ - # DockerServerImages: - # needs: - # - BuilderDebRelease - # - BuilderDebAarch64 - # runs-on: [self-hosted, ibm-vm] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - # - name: Check docker clickhouse/clickhouse-server building - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # DOCKER_REPO="${DOCKER_REPO:-docker.io}" - # python3 docker_server.py --release-type head --no-push \ - # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server - # python3 docker_server.py --release-type head --no-push \ - # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # # ########################################################################################## +# # # ##################################### SPECIAL BUILDS ##################################### +# # # ########################################################################################## +# # BuilderBinClangTidy: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_tidy +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinDarwin: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_darwin +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinAarch64: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinFreeBSD: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_freebsd +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinDarwinAarch64: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_darwin_aarch64 +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinPPC64: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_ppc64le +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinAmd64Compat: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_amd64_compat +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # BuilderBinAarch64V80Compat: +# # needs: [DockerHubPush, FastTest, StyleCheck] +# # runs-on: [self-hosted, builder] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/build_check +# # IMAGES_PATH=${{runner.temp}}/images_path +# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# # CACHES_PATH=${{runner.temp}}/../ccaches +# # BUILD_NAME=binary_aarch64_v80compat +# # EOF +# # - name: Download changed images +# # uses: actions/download-artifact@v3 +# # with: +# # name: changed_images +# # path: ${{ env.IMAGES_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # submodules: true +# # - name: Build +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# # - name: Upload build URLs to artifacts +# # if: ${{ success() || failure() }} +# # uses: actions/upload-artifact@v3 +# # with: +# # name: ${{ env.BUILD_URLS }} +# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ############################################################################################ +# # ##################################### Docker images ####################################### +# # ############################################################################################ +# # DockerServerImages: +# # needs: +# # - BuilderDebRelease +# # - BuilderDebAarch64 +# # runs-on: [self-hosted, ibm-vm] +# # steps: +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself +# # - name: Check docker clickhouse/clickhouse-server building +# # run: | +# # cd "$GITHUB_WORKSPACE/tests/ci" +# # DOCKER_REPO="${DOCKER_REPO:-docker.io}" +# # python3 docker_server.py --release-type head --no-push \ +# # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server +# # python3 docker_server.py --release-type head --no-push \ +# # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 45298a68b239..29aa0d75e28b 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -41,8 +41,8 @@ if [ -n "$MAKE_DEB" ]; then # We need to check if clickhouse-diagnostics is fine and build it ( cd /build/programs/diagnostics - make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" GITHUB_REPOSITORY="$github_repository" build + GITHUB_REPOSITORY="$github_repository" make test-no-docker + GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build mv clickhouse-diagnostics .. ) else From ba1f992f2a60b7e53e630f118460292efb4f1678 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 10:05:13 -0400 Subject: [PATCH 178/213] uncommented --- .github/workflows/pull_test.yaml | 7308 +++++++++++++++--------------- .github/workflows/pull_tests.yml | 2194 ++++----- 2 files changed, 4751 insertions(+), 4751 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index 33fcf42f0c93..e523c91be13f 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -1,335 +1,294 @@ -# name: PullTestCI +name: PullTestCI -# env: -# # Force the stdout and stderr streams to be unbuffered -# PYTHONUNBUFFERED: 1 +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 -# on: # yamllint disable-line rule:truthy -# push: -# # pull_request: -# # types: -# # - synchronize -# # - reopened -# # - opened -# # branches: -# # - master -# # paths-ignore: -# # - 'CHANGELOG.md' -# # - 'README.md' -# # - 'SECURITY.md' -# # - 'docker/docs/**' -# # - 'docs/**' -# # - 'utils/check-style/aspell-ignore/**' -# ########################################################################################## -# ##################################### SMALL CHECKS ####################################### -# ########################################################################################## -# jobs: -# # CheckLabels: -# # runs-on: [self-hosted, style-checker] -# # # Run the first check always, even if the CI is cancelled -# # if: ${{ always() }} -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Labels check -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 run_check.py -# # PythonUnitTests: -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Python unit tests -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # echo "Testing the main ci directory" -# # python3 -m unittest discover -s . -p '*_test.py' -# # for dir in *_lambda/; do -# # echo "Testing $dir" -# # python3 -m unittest discover -s "$dir" -p '*_test.py' -# # done -# DockerHubPushAarch64: -# # needs: CheckLabels -# runs-on: [self-hosted, test-only] -# steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Images check -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_images_check.py --suffix aarch64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images_aarch64 -# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json -# DockerHubPushAmd64: -# # needs: CheckLabels -# runs-on: [self-hosted, test-only] +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## +jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done + DockerHubPushAarch64: + # needs: CheckLabels + runs-on: [self-hosted, test-only] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + # needs: CheckLabels + runs-on: [self-hosted, test-only] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json + StyleCheck: + needs: DockerHubPush + runs-on: [self-hosted, builder] + # We need additional `&& ! cancelled()` to have the job being able to cancel + if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + steps: + - name: Set envs + run: echo "hi" + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{ runner.temp }}/style_check + # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test + # run: | + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +## CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] # steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Images check +# - name: Set envs # run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_images_check.py --suffix amd64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images_amd64 -# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json -# DockerHubPush: -# needs: [DockerHubPushAmd64, DockerHubPushAarch64] -# runs-on: [self-hosted, builder] -# steps: +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Download changed aarch64 images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images_aarch64 -# path: ${{ runner.temp }} -# - name: Download changed amd64 images +# - name: Download json reports # uses: actions/download-artifact@v3 # with: -# name: changed_images_amd64 -# path: ${{ runner.temp }} -# - name: Images check +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 # run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images -# path: ${{ runner.temp }}/changed_images.json -# StyleCheck: -# needs: DockerHubPush -# runs-on: [self-hosted, builder] -# # We need additional `&& ! cancelled()` to have the job being able to cancel -# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} -# steps: -# - name: Set envs -# run: echo "hi" -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{ runner.temp }}/style_check -# # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/fasttest -# # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # EOF -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.TEMP_PATH }} -# # - name: Fast Test -# # run: | -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ## CompatibilityCheckX86: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/compatibility_check -# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # EOF -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: CompatibilityCheckX86 -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # CompatibilityCheckAarch64: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/compatibility_check -# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # EOF -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: CompatibilityCheckAarch64 -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ######################################################################################### -# # #################################### ORDINARY BUILDS #################################### -# # ######################################################################################### -# BuilderDebRelease: -# needs: [DockerHubPush, FastTest] -# runs-on: [self-hosted, builder] +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_release +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir # EOF -# - name: Download changed images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# fetch-depth: 0 # for performance artifact -# submodules: true -# - name: Build +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # BuilderBinRelease: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_release -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebAarch64: +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### + BuilderDebRelease: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -340,19 +299,18 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_aarch64 +# BUILD_NAME=binary_release # EOF # - name: Download changed images # uses: actions/download-artifact@v3 # with: # name: changed_images -# path: ${{ runner.temp }}/images_path +# path: ${{ env.IMAGES_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true # submodules: true -# fetch-depth: 0 # for performance artifact # - name: Build # run: | # sudo rm -fr "$TEMP_PATH" @@ -371,7 +329,264 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebAsan: + BuilderDebAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_asan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebUBsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_ubsan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebTsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_tsan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebMsan: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_msan + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebDebug: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_debug + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Apply sparse checkout for contrib # in order to check that it doesn't break build + run: | + rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' + git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' + "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' + du -hs "$GITHUB_WORKSPACE/contrib" ||: + find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -382,7 +597,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_asan +# BUILD_NAME=binary_tidy # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -412,7 +627,48 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebUBsan: + BuilderBinDarwin: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -423,7 +679,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_ubsan +# BUILD_NAME=binary_aarch64 # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -453,7 +709,7 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebTsan: +# BuilderBinFreeBSD: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -464,7 +720,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_tsan +# BUILD_NAME=binary_freebsd # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -494,7 +750,48 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebMsan: + BuilderBinDarwinAarch64: + needs: [DockerHubPush, FastTest, StyleCheck] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -505,7 +802,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_msan +# BUILD_NAME=binary_ppc64le # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -535,7 +832,7 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderDebDebug: +# BuilderBinAmd64Compat: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -546,7 +843,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_debug +# BUILD_NAME=binary_amd64_compat # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -558,13 +855,6 @@ # with: # clear-repository: true # submodules: true -# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build -# run: | -# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' -# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' -# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' -# du -hs "$GITHUB_WORKSPACE/contrib" ||: -# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: # - name: Build # run: | # sudo rm -fr "$TEMP_PATH" @@ -583,51 +873,7 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # ########################################################################################## -# # ##################################### SPECIAL BUILDS ##################################### -# # ########################################################################################## -# # BuilderBinClangTidy: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_tidy -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwin: +# BuilderBinAarch64V80Compat: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] # steps: @@ -638,7 +884,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin +# BUILD_NAME=binary_aarch64_v80compat # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -668,454 +914,138 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAarch64: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinFreeBSD: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_freebsd -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# BuilderBinDarwinAarch64: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_darwin_aarch64 +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json # EOF -# - name: Download changed images +# - name: Download json reports # uses: actions/download-artifact@v3 # with: -# name: changed_images -# path: ${{ env.IMAGES_PATH }} +# path: ${{ env.REPORTS_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# submodules: true -# - name: Build +# - name: Report Builder # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" -# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# - name: Upload build URLs to artifacts -# if: ${{ success() || failure() }} -# uses: actions/upload-artifact@v3 -# with: -# name: ${{ env.BUILD_URLS }} -# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinPPC64: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_ppc64le -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAmd64Compat: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_amd64_compat -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAarch64V80Compat: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_aarch64_v80compat -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # ############################################################################################ -# # ##################################### Docker images ####################################### -# # ############################################################################################ -# DockerServerImages: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: # needs: -# - BuilderDebRelease -# - BuilderDebAarch64 +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy # runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} # steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself -# - name: Check docker clickhouse/clickhouse-server building +# - name: Report Builder # run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF # cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_server.py --release-type head --no-push \ -# --image-repo clickhouse/clickhouse-server --image-path docker/server -# python3 docker_server.py --release-type head --no-push \ -# --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper +# python3 build_report_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################ -# # ##################################### BUILD REPORTER ####################################### -# # ############################################################################################ -# # BuilderReport: -# # needs: -# # - BuilderBinRelease -# # - BuilderDebAarch64 -# # - BuilderDebAsan -# # - BuilderDebDebug -# # - BuilderDebMsan -# # - BuilderDebRelease -# # - BuilderDebTsan -# # - BuilderDebUBsan -# # runs-on: [self-hosted, style-checker] -# # if: ${{ success() || failure() }} -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # CHECK_NAME=ClickHouse build check -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # TEMP_PATH=${{runner.temp}}/report_check -# # NEEDS_DATA_PATH=${{runner.temp}}/needs.json -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Report Builder -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cat > "$NEEDS_DATA_PATH" << 'EOF' -# # ${{ toJSON(needs) }} -# # EOF -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 build_report_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # BuilderSpecialReport: -# # needs: -# # - BuilderBinAarch64 -# # - BuilderBinDarwin -# # - BuilderBinDarwinAarch64 -# # - BuilderBinFreeBSD -# # - BuilderBinPPC64 -# # - BuilderBinAmd64Compat -# # - BuilderBinAarch64V80Compat -# # - BuilderBinClangTidy -# # runs-on: [self-hosted, style-checker] -# # if: ${{ success() || failure() }} -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/report_check -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=ClickHouse special build check -# # NEEDS_DATA_PATH=${{runner.temp}}/needs.json -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Report Builder -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cat > "$NEEDS_DATA_PATH" << 'EOF' -# # ${{ toJSON(needs) }} -# # EOF -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 build_report_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################ -# # #################################### INSTALL PACKAGES ###################################### -# # ############################################################################################ -# # InstallPackagesTestRelease: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/test_install -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Install packages (amd64) -# # REPO_COPY=${{runner.temp}}/test_install/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Test packages installation -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 install_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # InstallPackagesTestAarch64: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, style-checker-aarch64] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/test_install -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Install packages (arm64) -# # REPO_COPY=${{runner.temp}}/test_install/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Test packages installation -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 install_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ########################### FUNCTIONAl STATELESS TESTS ####################################### -# # ############################################################################################## -# FunctionalStatelessTestRelease: +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: # needs: [BuilderDebRelease] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, style-checker] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_release +# TEMP_PATH=${{runner.temp}}/test_install # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (release) -# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse -# KILL_TIMEOUT=10800 +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1125,711 +1055,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Test packages installation # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 install_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseDatabaseReplicated0: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseDatabaseReplicated1: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseDatabaseReplicated2: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseDatabaseReplicated3: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_database_replicated -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, DatabaseReplicated) -# # REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseWideParts: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_wide_parts -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, wide parts enabled) -# # REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse -# # KILL_TIMEOUT=10800 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseAnalyzer: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_analyzer -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, analyzer) -# # REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse -# # KILL_TIMEOUT=10800 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseS3_0: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=2 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestReleaseS3_1: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (release, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=2 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug0: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug1: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug2: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug3: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug4: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Debug5: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=5 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Tsan0: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Tsan1: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Tsan2: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Tsan3: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestS3Tsan4: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan, s3 storage) -# # REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestAarch64: +# InstallPackagesTestAarch64: # needs: [BuilderDebAarch64] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, style-checker-aarch64] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_release +# TEMP_PATH=${{runner.temp}}/test_install # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (aarch64) -# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse -# KILL_TIMEOUT=10800 +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -1839,30 +1088,67 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Test packages installation # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 install_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestAsan0: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## + FunctionalStatelessTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (asan) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=0 # RUN_BY_HASH_TOTAL=4 @@ -1888,17 +1174,17 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestAsan1: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (asan) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=1 # RUN_BY_HASH_TOTAL=4 @@ -1924,92 +1210,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestAsan2: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (asan) -# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestAsan3: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (asan) -# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestTsan0: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_tsan +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan) -# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=5 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2032,20 +1246,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestTsan1: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_tsan +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan) -# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=5 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2068,20 +1282,18 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestTsan2: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_tsan +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (tsan) -# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2104,92 +1316,18 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestTsan3: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan) -# # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestTsan4: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (tsan) -# # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestUBsan0: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# TEMP_PATH=${{runner.temp}}/stateless_analyzer # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (ubsan) -# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=2 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2212,56 +1350,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestUBsan1: -# # needs: [BuilderDebUBsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_ubsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (ubsan) -# # REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=2 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan0: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=2 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2284,20 +1386,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan1: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse # KILL_TIMEOUT=10800 # RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=2 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2320,19 +1422,19 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan2: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_memory +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (msan) -# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_NUM=0 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -2356,128 +1458,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestMsan3: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_memory -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (msan) -# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestMsan4: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_memory -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (msan) -# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestMsan5: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_memory -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (msan) -# # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=5 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestDebug0: +# FunctionalStatelessTestS3Debug1: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=5 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2500,20 +1494,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestDebug1: +# FunctionalStatelessTestS3Debug2: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=5 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2536,20 +1530,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestDebug2: +# FunctionalStatelessTestS3Debug3: # needs: [BuilderDebDebug] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateless_debug +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateless tests (debug) -# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=5 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2572,172 +1566,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestDebug3: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug) -# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestDebug4: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests (debug) -# # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=5 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatelessTestFlakyCheck: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateless_flaky_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateless tests flaky check (asan) -# # REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # TestsBugfixCheck: -# # needs: [CheckLabels, StyleCheck] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/tests_bugfix_check -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=tests bugfix validate check -# # KILL_TIMEOUT=3600 -# # REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Bugfix test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" - -# # TEMP_PATH="${TEMP_PATH}/integration" \ -# # REPORTS_PATH="${REPORTS_PATH}/integration" \ -# # python3 integration_test_check.py "Integration $CHECK_NAME" \ -# # --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - -# # TEMP_PATH="${TEMP_PATH}/stateless" \ -# # REPORTS_PATH="${REPORTS_PATH}/stateless" \ -# # python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ -# # --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - -# # python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ############################ FUNCTIONAl STATEFUL TESTS ####################################### -# # ############################################################################################## -# FunctionalStatefulTestRelease: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_release +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (release) -# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2760,18 +1602,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestAarch64: -# needs: [BuilderDebAarch64] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_release +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (aarch64) -# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2794,18 +1638,20 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestAsan: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_debug +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (asan) -# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2828,18 +1674,2349 @@ # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestTsan: +# FunctionalStatelessTestS3Tsan1: # needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (aarch64) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (asan) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (asan) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (tsan) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestUBsan0: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (ubsan) + REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan1: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan0: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan1: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan2: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (msan) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug0: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug1: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug2: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=5 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## + FunctionalStatefulTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (release) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (aarch64) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (asan) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestTsan: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (tsan) + REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_msan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (msan) + REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (ubsan) + REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (debug) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## + StressTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (asan) + REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestTsan: + needs: [BuilderDebTsan] + # func testers have 16 cores + 128 GB memory + # while stress testers have 36 cores + 72 memory + # It would be better to have something like 32 + 128, + # but such servers almost unavailable as spot instances. + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (tsan) + REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (msan) + REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_undefined + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (ubsan) + REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (debug) + REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# + IntegrationTestsAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan2: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan3: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_tsan +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (tsan) -# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2849,31 +4026,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestMsan: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, builder] +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_msan +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (msan) -# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2883,31 +4061,102 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestUBsan: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, builder] + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# TEMP_PATH=${{runner.temp}}/integration_tests_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (ubsan) -# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2917,31 +4166,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatefulTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, builder] +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stateful_debug +# TEMP_PATH=${{runner.temp}}/integration_tests_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stateful tests (debug) -# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# KILL_TIMEOUT=3600 +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -2951,244 +4201,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Functional test +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # # Parallel replicas -# # FunctionalStatefulTestDebugParallelReplicas: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (debug, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatefulTestUBsanParallelReplicas: -# # needs: [BuilderDebUBsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_ubsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatefulTestMsanParallelReplicas: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_msan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (msan, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatefulTestTsanParallelReplicas: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (tsan, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatefulTestAsanParallelReplicas: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (asan, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # FunctionalStatefulTestReleaseParallelReplicas: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/stateful_release -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Stateful tests (release, ParallelReplicas) -# # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse -# # KILL_TIMEOUT=3600 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Functional test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ######################################### STRESS TESTS ####################################### -# # ############################################################################################## -# StressTestAsan: +# IntegrationTestsFlakyCheck: # needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_asan +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (asan) -# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3198,34 +4234,33 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Integration test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 integration_test_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestTsan: -# needs: [BuilderDebTsan] -# # func testers have 16 cores + 128 GB memory -# # while stress testers have 36 cores + 72 memory -# # It would be better to have something like 32 + 128, -# # but such servers almost unavailable as spot instances. -# runs-on: [self-hosted, builder] +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_thread +# TEMP_PATH=${{runner.temp}}/unit_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (tsan) -# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3235,30 +4270,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Unit test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 unit_tests_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestMsan: -# needs: [BuilderDebMsan] -# runs-on: [self-hosted, builder] +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_memory +# TEMP_PATH=${{runner.temp}}/unit_tests_asan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (msan) -# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3268,30 +4303,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Unit test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 unit_tests_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestUBsan: -# needs: [BuilderDebUBsan] -# runs-on: [self-hosted, builder] +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_undefined +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (ubsan) -# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3301,30 +4336,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Unit test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 unit_tests_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# StressTestDebug: -# needs: [BuilderDebDebug] -# runs-on: [self-hosted, builder] +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/stress_debug +# TEMP_PATH=${{runner.temp}}/unit_tests_msan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Stress test (debug) -# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3334,339 +4369,68 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Stress test +# - name: Unit test # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 stress_check.py "$CHECK_NAME" +# python3 unit_tests_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ######################################### UPGRADE CHECK ###################################### -# # ############################################################################################## -# # UpgradeCheckAsan: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/upgrade_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Upgrade check (asan) -# # REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Upgrade check -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 upgrade_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UpgradeCheckTsan: -# # needs: [BuilderDebTsan] -# # # same as for stress test with tsan -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/upgrade_thread -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Upgrade check (tsan) -# # REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Upgrade check -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 upgrade_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UpgradeCheckMsan: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/upgrade_memory -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Upgrade check (msan) -# # REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Upgrade check -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 upgrade_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UpgradeCheckDebug: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/upgrade_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Upgrade check (debug) -# # REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Upgrade check -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 upgrade_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ##################################### AST FUZZERS ############################################ -# # ############################################################################################## -# # ASTFuzzerTestAsan: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=AST fuzzer (asan) -# # REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Fuzzer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 ast_fuzzer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ASTFuzzerTestTsan: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=AST fuzzer (tsan) -# # REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Fuzzer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 ast_fuzzer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ASTFuzzerTestUBSan: -# # needs: [BuilderDebUBsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=AST fuzzer (ubsan) -# # REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Fuzzer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 ast_fuzzer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ASTFuzzerTestMSan: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=AST fuzzer (msan) -# # REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Fuzzer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 ast_fuzzer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ASTFuzzerTestDebug: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=AST fuzzer (debug) -# # REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Fuzzer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 ast_fuzzer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################# -# # ############################# INTEGRATION TESTS ############################################# -# # ############################################################################################# -# IntegrationTestsAsan0: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3676,32 +4440,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan1: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3711,32 +4475,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsAsan2: -# needs: [BuilderDebAsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (asan) -# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3746,137 +4510,67 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsAsan3: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (asan) -# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsAsan4: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (asan) -# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsAsan5: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (asan) -# # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse -# # RUN_BY_HASH_NUM=5 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan0: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3886,32 +4580,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan1: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3921,32 +4615,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan2: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=2 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3956,32 +4650,32 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsTsan3: -# needs: [BuilderDebTsan] -# runs-on: [self-hosted, builder] +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# TEMP_PATH=${{runner.temp}}/performance_comparison # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (tsan) -# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse # RUN_BY_HASH_NUM=3 -# RUN_BY_HASH_TOTAL=6 +# RUN_BY_HASH_TOTAL=4 # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -3991,102 +4685,33 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: Performance Comparison # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 performance_comparison_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsTsan4: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (tsan) -# # REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# # RUN_BY_HASH_NUM=4 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsTsan5: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (tsan) -# # REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse -# # RUN_BY_HASH_NUM=5 -# # RUN_BY_HASH_TOTAL=6 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease0: +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: # needs: [BuilderDebRelease] -# runs-on: [self-hosted, builder] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release +# TEMP_PATH=${{runner.temp}}/sqlancer_release # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=0 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4096,32 +4721,30 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: SQLancer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 sqlancer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# IntegrationTestsRelease1: -# needs: [BuilderDebRelease] -# runs-on: [self-hosted, builder] +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] # steps: # - name: Set envs # run: | # cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/integration_tests_release +# TEMP_PATH=${{runner.temp}}/sqlancer_debug # REPORTS_PATH=${{runner.temp}}/reports_dir -# CHECK_NAME=Integration tests (release) -# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# RUN_BY_HASH_NUM=1 -# RUN_BY_HASH_TOTAL=4 +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse # EOF # - name: Download json reports # uses: actions/download-artifact@v3 @@ -4131,799 +4754,176 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# - name: Integration test +# - name: SQLancer # run: | # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" # cd "$REPO_COPY/tests/ci" -# python3 integration_test_check.py "$CHECK_NAME" +# python3 sqlancer_check.py "$CHECK_NAME" # - name: Cleanup # if: always() # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsRelease2: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_release -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (release) -# # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsRelease3: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_release -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests (release) -# # REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # IntegrationTestsFlakyCheck: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Integration tests flaky check (asan) -# # REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Integration test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 integration_test_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################# -# # #################################### UNIT TESTS ############################################# -# # ############################################################################################# -# # UnitTestsAsan: -# # needs: [BuilderDebAsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/unit_tests_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Unit tests (asan) -# # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Unit test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 unit_tests_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UnitTestsReleaseClang: -# # needs: [BuilderBinRelease] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/unit_tests_asan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Unit tests (release-clang) -# # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Unit test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 unit_tests_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UnitTestsTsan: -# # needs: [BuilderDebTsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/unit_tests_tsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Unit tests (tsan) -# # REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Unit test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 unit_tests_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UnitTestsMsan: -# # needs: [BuilderDebMsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/unit_tests_msan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Unit tests (msan) -# # REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Unit test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 unit_tests_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # UnitTestsUBsan: -# # needs: [BuilderDebUBsan] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/unit_tests_ubsan -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Unit tests (ubsan) -# # REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Unit test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 unit_tests_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################# -# # #################################### PERFORMANCE TESTS ###################################### -# # ############################################################################################# -# # PerformanceComparisonX86-0: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonX86-1: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonX86-2: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonX86-3: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, stress-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonAarch-0: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, func-tester-aarch64] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison Aarch64 -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=0 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonAarch-1: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, func-tester-aarch64] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison Aarch64 -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=1 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonAarch-2: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, func-tester-aarch64] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison Aarch64 -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=2 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # PerformanceComparisonAarch-3: -# # needs: [BuilderDebAarch64] -# # runs-on: [self-hosted, func-tester-aarch64] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/performance_comparison -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Performance Comparison Aarch64 -# # REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse -# # RUN_BY_HASH_NUM=3 -# # RUN_BY_HASH_TOTAL=4 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Performance Comparison -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 performance_comparison_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################## -# # ###################################### SQLANCER FUZZERS ###################################### -# # ############################################################################################## -# # SQLancerTestRelease: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/sqlancer_release -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=SQLancer (release) -# # REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: SQLancer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 sqlancer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # SQLancerTestDebug: -# # needs: [BuilderDebDebug] -# # runs-on: [self-hosted, fuzzer-unit-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/sqlancer_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=SQLancer (debug) -# # REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v3 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: SQLancer -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 sqlancer_check.py "$CHECK_NAME" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # ############################################################################################# -# # ###################################### JEPSEN TESTS ######################################### -# # ############################################################################################# -# # Jepsen: -# # # This is special test NOT INCLUDED in FinishCheck -# # # When it's skipped, all dependent tasks will be skipped too. -# # # DO NOT add it there -# # if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') -# # needs: [BuilderBinRelease] -# # uses: ./.github/workflows/jepsen.yml -# # FinishCheck: -# # needs: -# # - StyleCheck -# # - DockerHubPush -# # - DockerServerImages -# # - CheckLabels -# # - BuilderReport -# # - BuilderSpecialReport -# # - FastTest -# # - FunctionalStatelessTestDebug0 -# # - FunctionalStatelessTestDebug1 -# # - FunctionalStatelessTestDebug2 -# # - FunctionalStatelessTestDebug3 -# # - FunctionalStatelessTestDebug4 -# # - FunctionalStatelessTestRelease -# # - FunctionalStatelessTestReleaseDatabaseReplicated0 -# # - FunctionalStatelessTestReleaseDatabaseReplicated1 -# # - FunctionalStatelessTestReleaseDatabaseReplicated2 -# # - FunctionalStatelessTestReleaseDatabaseReplicated3 -# # - FunctionalStatelessTestReleaseWideParts -# # - FunctionalStatelessTestReleaseAnalyzer -# # - FunctionalStatelessTestAarch64 -# # - FunctionalStatelessTestAsan0 -# # - FunctionalStatelessTestAsan1 -# # - FunctionalStatelessTestAsan2 -# # - FunctionalStatelessTestAsan3 -# # - FunctionalStatelessTestTsan0 -# # - FunctionalStatelessTestTsan1 -# # - FunctionalStatelessTestTsan2 -# # - FunctionalStatelessTestTsan3 -# # - FunctionalStatelessTestTsan4 -# # - FunctionalStatelessTestMsan0 -# # - FunctionalStatelessTestMsan1 -# # - FunctionalStatelessTestMsan2 -# # - FunctionalStatelessTestMsan3 -# # - FunctionalStatelessTestMsan4 -# # - FunctionalStatelessTestMsan5 -# # - FunctionalStatelessTestUBsan0 -# # - FunctionalStatelessTestUBsan1 -# # - FunctionalStatefulTestDebug -# # - FunctionalStatefulTestRelease -# # - FunctionalStatefulTestAarch64 -# # - FunctionalStatefulTestAsan -# # - FunctionalStatefulTestTsan -# # - FunctionalStatefulTestMsan -# # - FunctionalStatefulTestUBsan -# # - FunctionalStatelessTestReleaseS3_0 -# # - FunctionalStatelessTestReleaseS3_1 -# # - FunctionalStatelessTestS3Debug0 -# # - FunctionalStatelessTestS3Debug1 -# # - FunctionalStatelessTestS3Debug2 -# # - FunctionalStatelessTestS3Debug4 -# # - FunctionalStatelessTestS3Debug5 -# # - FunctionalStatelessTestS3Tsan0 -# # - FunctionalStatelessTestS3Tsan1 -# # - FunctionalStatelessTestS3Tsan2 -# # - FunctionalStatelessTestS3Tsan4 -# # - StressTestDebug -# # - StressTestAsan -# # - StressTestTsan -# # - StressTestMsan -# # - StressTestUBsan -# # - ASTFuzzerTestDebug -# # - ASTFuzzerTestAsan -# # - ASTFuzzerTestTsan -# # - ASTFuzzerTestMSan -# # - ASTFuzzerTestUBSan -# # - IntegrationTestsAsan0 -# # - IntegrationTestsAsan1 -# # - IntegrationTestsAsan2 -# # - IntegrationTestsAsan3 -# # - IntegrationTestsAsan4 -# # - IntegrationTestsAsan5 -# # - IntegrationTestsRelease0 -# # - IntegrationTestsRelease1 -# # - IntegrationTestsRelease2 -# # - IntegrationTestsRelease3 -# # - IntegrationTestsTsan0 -# # - IntegrationTestsTsan1 -# # - IntegrationTestsTsan2 -# # - IntegrationTestsTsan3 -# # - IntegrationTestsTsan4 -# # - IntegrationTestsTsan5 -# # - PerformanceComparisonX86-0 -# # - PerformanceComparisonX86-1 -# # - PerformanceComparisonX86-2 -# # - PerformanceComparisonX86-3 -# # - PerformanceComparisonAarch-0 -# # - PerformanceComparisonAarch-1 -# # - PerformanceComparisonAarch-2 -# # - PerformanceComparisonAarch-3 -# # - UnitTestsAsan -# # - UnitTestsTsan -# # - UnitTestsMsan -# # - UnitTestsUBsan -# # - UnitTestsReleaseClang -# # - CompatibilityCheckX86 -# # - CompatibilityCheckAarch64 -# # - IntegrationTestsFlakyCheck -# # - SQLancerTestRelease -# # - SQLancerTestDebug -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Finish label -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 finish_check.py -# # python3 merge_pr.py --check-approved -# # ############################################################################################## -# # ########################### SQLLOGIC TEST ################################################### -# # ############################################################################################## -# # SQLLogicTestRelease: -# # needs: [BuilderDebRelease] -# # runs-on: [self-hosted, func-tester] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/sqllogic_debug -# # REPORTS_PATH=${{runner.temp}}/reports_dir -# # CHECK_NAME=Sqllogic test (release) -# # REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse -# # KILL_TIMEOUT=10800 -# # EOF -# # - name: Download json reports -# # uses: actions/download-artifact@v2 -# # with: -# # path: ${{ env.REPORTS_PATH }} -# # - name: Clear repository -# # run: | -# # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" -# # - name: Check out repository code -# # uses: actions/checkout@v2 -# # - name: Sqllogic test -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" -# # python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" \ No newline at end of file +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" \ No newline at end of file diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 0846e4cbe893..c3b4a6f17e2e 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -1,313 +1,4 @@ -# # name: PullclangCI - -# # env: -# # # Force the stdout and stderr streams to be unbuffered -# # PYTHONUNBUFFERED: 1 - -# # on: # yamllint disable-line rule:truthy -# # push: -# # # pull_request: -# # # types: -# # # - synchronize -# # # - reopened -# # # - opened -# # # branches: -# # # - master -# # # paths-ignore: -# # # - 'CHANGELOG.md' -# # # - 'README.md' -# # # - 'SECURITY.md' -# # # - 'docker/docs/**' -# # # - 'docs/**' -# # # - 'utils/check-style/aspell-ignore/**' -# # jobs: -# # DockerHubPushAarch64: -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: test -# # run: env -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Images check -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 docker_images_check.py --suffix aarch64 -# # - name: Upload images files to artifacts -# # uses: actions/upload-artifact@v3 -# # with: -# # name: changed_images_aarch64 -# # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json -# # DockerHubPushAmd64: -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Images check -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 docker_images_check.py --suffix amd64 -# # - name: Upload images files to artifacts -# # uses: actions/upload-artifact@v3 -# # with: -# # name: changed_images_amd64 -# # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json -# # DockerHubPush: -# # needs: [DockerHubPushAmd64, DockerHubPushAarch64] -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Download changed aarch64 images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images_aarch64 -# # path: ${{ runner.temp }} -# # - name: Download changed amd64 images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images_amd64 -# # path: ${{ runner.temp }} -# # - name: Images check -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 -# # - name: Upload images files to artifacts -# # uses: actions/upload-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ runner.temp }}/changed_images.json -# # BuilderBinClangTidy: -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_tidy -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinDarwin: -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_darwin -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinDarwinAarch64: -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_darwin_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebRelease: -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_release -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # fetch-depth: 0 # for performance artifact -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" -# # BuilderDebAarch64: -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, test-only] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ runner.temp }}/images_path -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # fetch-depth: 0 # for performance artifact -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # MarkReleaseReady: -# # needs: -# # - BuilderBinDarwin -# # - BuilderBinDarwinAarch64 -# # - BuilderDebRelease -# # - BuilderDebAarch64 -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Mark Commit Release Ready -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 mark_release_ready.py - - -# name: docker-check +# name: PullclangCI # env: # # Force the stdout and stderr streams to be unbuffered @@ -328,44 +19,13 @@ # # - 'SECURITY.md' # # - 'docker/docs/**' # # - 'docs/**' -# # - 'utils/check-style/aspell-ignore/**' -# ########################################################################################## -# ##################################### SMALL CHECKS ####################################### -# ########################################################################################## +# # - 'utils/check-style/aspell-ignore/**' # jobs: -# # CheckLabels: -# # runs-on: [self-hosted, style-checker] -# # # Run the first check always, even if the CI is cancelled -# # if: ${{ always() }} -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Labels check -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # python3 run_check.py -# # PythonUnitTests: -# # runs-on: [self-hosted, style-checker] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Python unit tests -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # echo "Testing the main ci directory" -# # python3 -m unittest discover -s . -p '*_test.py' -# # for dir in *_lambda/; do -# # echo "Testing $dir" -# # python3 -m unittest discover -s "$dir" -p '*_test.py' -# # done # DockerHubPushAarch64: -# # needs: CheckLabels -# runs-on: [self-hosted, style-checker] +# runs-on: [self-hosted, ibm-vm] # steps: +# - name: test +# run: env # - name: Check out repository code # uses: ClickHouse/checkout@v1 # with: @@ -380,8 +40,7 @@ # name: changed_images_aarch64 # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # DockerHubPushAmd64: -# # needs: CheckLabels -# runs-on: [self-hosted, style-checker] +# runs-on: [self-hosted, ibm-vm] # steps: # - name: Check out repository code # uses: ClickHouse/checkout@v1 @@ -398,7 +57,7 @@ # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json # DockerHubPush: # needs: [DockerHubPushAmd64, DockerHubPushAarch64] -# runs-on: [self-hosted, style-checker] +# runs-on: [self-hosted, ibm-vm] # steps: # - name: Check out repository code # uses: ClickHouse/checkout@v1 @@ -423,140 +82,9 @@ # with: # name: changed_images # path: ${{ runner.temp }}/changed_images.json -# # StyleCheck: -# # needs: DockerHubPush -# # runs-on: [self-hosted, style-checker] -# # # We need additional `&& ! cancelled()` to have the job being able to cancel -# # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{ runner.temp }}/style_check -# # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/fasttest -# # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # EOF -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.TEMP_PATH }} -# # - name: Fast Test -# # run: | -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # ## CompatibilityCheckX86: -# # # needs: [BuilderDebRelease] -# # # runs-on: [self-hosted, style-checker] -# # # steps: -# # # - name: Set envs -# # # run: | -# # # cat >> "$GITHUB_ENV" << 'EOF' -# # # TEMP_PATH=${{runner.temp}}/compatibility_check -# # # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # # REPORTS_PATH=${{runner.temp}}/reports_dir -# # # EOF -# # # - name: Check out repository code -# # # uses: ClickHouse/checkout@v1 -# # # with: -# # # clear-repository: true -# # # - name: Download json reports -# # # uses: actions/download-artifact@v3 -# # # with: -# # # path: ${{ env.REPORTS_PATH }} -# # # - name: CompatibilityCheckX86 -# # # run: | -# # # sudo rm -fr "$TEMP_PATH" -# # # mkdir -p "$TEMP_PATH" -# # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions -# # # - name: Cleanup -# # # if: always() -# # # run: | -# # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # # sudo rm -fr "$TEMP_PATH" -# # # CompatibilityCheckAarch64: -# # # needs: [BuilderDebAarch64] -# # # runs-on: [self-hosted, style-checker] -# # # steps: -# # # - name: Set envs -# # # run: | -# # # cat >> "$GITHUB_ENV" << 'EOF' -# # # TEMP_PATH=${{runner.temp}}/compatibility_check -# # # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse -# # # REPORTS_PATH=${{runner.temp}}/reports_dir -# # # EOF -# # # - name: Check out repository code -# # # uses: ClickHouse/checkout@v1 -# # # with: -# # # clear-repository: true -# # # - name: Download json reports -# # # uses: actions/download-artifact@v3 -# # # with: -# # # path: ${{ env.REPORTS_PATH }} -# # # - name: CompatibilityCheckAarch64 -# # # run: | -# # # sudo rm -fr "$TEMP_PATH" -# # # mkdir -p "$TEMP_PATH" -# # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc -# # # - name: Cleanup -# # # if: always() -# # # run: | -# # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # # sudo rm -fr "$TEMP_PATH" -# # # ######################################################################################### -# # # #################################### ORDINARY BUILDS #################################### -# # # ######################################################################################### -# BuilderDebRelease: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# needs: [DockerHubPush] -# runs-on: [self-hosted, builder] +# BuilderBinClangTidy: +# needs: [DockerHubPush] +# runs-on: [self-hosted, ibm-vm] # steps: # - name: Set envs # run: | @@ -565,7 +93,7 @@ # IMAGES_PATH=${{runner.temp}}/images_path # REPO_COPY=${{runner.temp}}/build_check/ClickHouse # CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=package_release +# BUILD_NAME=binary_tidy # EOF # - name: Download changed images # uses: actions/download-artifact@v3 @@ -576,7 +104,6 @@ # uses: ClickHouse/checkout@v1 # with: # clear-repository: true -# fetch-depth: 0 # for performance artifact # submodules: true # - name: Build # run: | @@ -595,659 +122,1132 @@ # run: | # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwin: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebRelease: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, test-only] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # MarkReleaseReady: + # needs: + # - BuilderBinDarwin + # - BuilderBinDarwinAarch64 + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Mark Commit Release Ready + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 mark_release_ready.py + + +name: docker-check + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## +jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done + DockerHubPushAarch64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json +# StyleCheck: +# needs: DockerHubPush +# runs-on: [self-hosted, style-checker] +# # We need additional `&& ! cancelled()` to have the job being able to cancel +# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{ runner.temp }}/style_check +# ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_release -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebAarch64: -# # # needs: [DockerHubPush, FastTest, StyleCheck] -# # needs: [DockerHubPush] -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ runner.temp }}/images_path -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # fetch-depth: 0 # for performance artifact -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebAsan: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] +# FastTest: +# needs: DockerHubPush +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/fasttest +# REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.TEMP_PATH }} +# - name: Fast Test +# run: | +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ## CompatibilityCheckX86: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, style-checker] # # steps: # # - name: Set envs # # run: | # # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_asan +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir # # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} # # - name: Check out repository code # # uses: ClickHouse/checkout@v1 # # with: # # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebUBsan: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_ubsan -# # EOF -# # - name: Download changed images +# # - name: Download json reports # # uses: actions/download-artifact@v3 # # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckX86 # # run: | # # sudo rm -fr "$TEMP_PATH" # # mkdir -p "$TEMP_PATH" # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions # # - name: Cleanup # # if: always() # # run: | # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebTsan: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_tsan -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | # # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebMsan: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] +# # CompatibilityCheckAarch64: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, style-checker] # # steps: # # - name: Set envs # # run: | # # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_msan +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir # # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} # # - name: Check out repository code # # uses: ClickHouse/checkout@v1 # # with: # # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderDebDebug: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=package_debug -# # EOF -# # - name: Download changed images +# # - name: Download json reports # # uses: actions/download-artifact@v3 # # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Apply sparse checkout for contrib # in order to check that it doesn't break build -# # run: | -# # rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' -# # git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' -# # "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' -# # du -hs "$GITHUB_WORKSPACE/contrib" ||: -# # find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: -# # - name: Build +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckAarch64 # # run: | # # sudo rm -fr "$TEMP_PATH" # # mkdir -p "$TEMP_PATH" # # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc # # - name: Cleanup # # if: always() # # run: | # # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # # ########################################################################################## -# # # ##################################### SPECIAL BUILDS ##################################### -# # # ########################################################################################## -# # BuilderBinClangTidy: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_tidy -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | # # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinDarwin: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_darwin -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAarch64: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinFreeBSD: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_freebsd -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinDarwinAarch64: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_darwin_aarch64 -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinPPC64: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_ppc64le -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAmd64Compat: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_amd64_compat -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # BuilderBinAarch64V80Compat: -# # needs: [DockerHubPush, FastTest, StyleCheck] -# # runs-on: [self-hosted, builder] -# # steps: -# # - name: Set envs -# # run: | -# # cat >> "$GITHUB_ENV" << 'EOF' -# # TEMP_PATH=${{runner.temp}}/build_check -# # IMAGES_PATH=${{runner.temp}}/images_path -# # REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# # CACHES_PATH=${{runner.temp}}/../ccaches -# # BUILD_NAME=binary_aarch64_v80compat -# # EOF -# # - name: Download changed images -# # uses: actions/download-artifact@v3 -# # with: -# # name: changed_images -# # path: ${{ env.IMAGES_PATH }} -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # submodules: true -# # - name: Build -# # run: | -# # sudo rm -fr "$TEMP_PATH" -# # mkdir -p "$TEMP_PATH" -# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" -# # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" -# # - name: Upload build URLs to artifacts -# # if: ${{ success() || failure() }} -# # uses: actions/upload-artifact@v3 -# # with: -# # name: ${{ env.BUILD_URLS }} -# # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# # ############################################################################################ -# # ##################################### Docker images ####################################### -# # ############################################################################################ -# # DockerServerImages: -# # needs: -# # - BuilderDebRelease -# # - BuilderDebAarch64 -# # runs-on: [self-hosted, ibm-vm] -# # steps: -# # - name: Check out repository code -# # uses: ClickHouse/checkout@v1 -# # with: -# # clear-repository: true -# # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself -# # - name: Check docker clickhouse/clickhouse-server building -# # run: | -# # cd "$GITHUB_WORKSPACE/tests/ci" -# # DOCKER_REPO="${DOCKER_REPO:-docker.io}" -# # python3 docker_server.py --release-type head --no-push \ -# # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server -# # python3 docker_server.py --release-type head --no-push \ -# # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper -# # - name: Cleanup -# # if: always() -# # run: | -# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: -# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: -# # sudo rm -fr "$TEMP_PATH" +# # ######################################################################################### +# # #################################### ORDINARY BUILDS #################################### +# # ######################################################################################### + BuilderDebRelease: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAarch64: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush] + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_asan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebUBsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_ubsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebTsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_tsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebMsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_msan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebDebug: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_debug +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ########################################################################################## +# # ##################################### SPECIAL BUILDS ##################################### +# # ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + # DockerServerImages: + # needs: + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # DOCKER_REPO="${DOCKER_REPO:-docker.io}" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" From 41f57762c0a00d61ecf482d6d8e6cadf8f7e1fca Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 10:22:51 -0400 Subject: [PATCH 179/213] CHECK FOR BUILDDEBRELEASE --- .github/workflows/pull_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index c3b4a6f17e2e..e8830b254e6d 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -596,6 +596,7 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] From a2307469df22c10d21f6c5d2bfba1296083ff8d5 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 11:21:06 -0400 Subject: [PATCH 180/213] CHECK FOR binary image --- .github/workflows/pull_tests.yml | 1 + docker/packager/binary/build.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index e8830b254e6d..dadb484c449e 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -423,6 +423,7 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json + # StyleCheck: # needs: DockerHubPush # runs-on: [self-hosted, style-checker] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 29aa0d75e28b..28c584f984df 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -x -e -export github_repository=${GITHUB_REPOSITORY:-ClickHouse/ClickHouse} +github_repository=${GITHUB_REPOSITORY} exec &> >(ts) @@ -41,7 +41,7 @@ if [ -n "$MAKE_DEB" ]; then # We need to check if clickhouse-diagnostics is fine and build it ( cd /build/programs/diagnostics - GITHUB_REPOSITORY="$github_repository" make test-no-docker + GITHUB_REPOSITORY="${github_repository}" make test-no-docker GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build mv clickhouse-diagnostics .. ) From c8c8fc336d5c93779a9604bf6ce12fda2fceadbb Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 12:20:34 -0400 Subject: [PATCH 181/213] CHECK FOR binary image --- .github/workflows/pull_test.yaml | 2732 ++++++++--------- .github/workflows/pull_tests.yml | 14 +- .../cmd/clickhouse-diagnostics/main.go | 2 +- programs/diagnostics/cmd/collect.go | 18 +- programs/diagnostics/cmd/help.go | 8 +- .../diagnostics/cmd/params/params_test.go | 4 +- .../internal/collectors/clickhouse/config.go | 10 +- .../diagnostics/internal/platform/manager.go | 4 +- 8 files changed, 1396 insertions(+), 1396 deletions(-) diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml index e523c91be13f..e84954a0c215 100644 --- a/.github/workflows/pull_test.yaml +++ b/.github/workflows/pull_test.yaml @@ -1,29 +1,29 @@ -name: PullTestCI +# name: PullTestCI -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 -on: # yamllint disable-line rule:truthy - push: - # pull_request: - # types: - # - synchronize - # - reopened - # - opened - # branches: - # - master - # paths-ignore: - # - 'CHANGELOG.md' - # - 'README.md' - # - 'SECURITY.md' - # - 'docker/docs/**' - # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' -########################################################################################## -##################################### SMALL CHECKS ####################################### -########################################################################################## -jobs: +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# ########################################################################################## +# ##################################### SMALL CHECKS ####################################### +# ########################################################################################## +# jobs: # CheckLabels: # runs-on: [self-hosted, style-checker] # # Run the first check always, even if the CI is cancelled @@ -53,75 +53,75 @@ jobs: # echo "Testing $dir" # python3 -m unittest discover -s "$dir" -p '*_test.py' # done - DockerHubPushAarch64: - # needs: CheckLabels - runs-on: [self-hosted, test-only] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - # needs: CheckLabels - runs-on: [self-hosted, test-only] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, builder] - # We need additional `&& ! cancelled()` to have the job being able to cancel - if: ${{ success() || failure() || ( always() && ! cancelled() ) }} - steps: - - name: Set envs - run: echo "hi" + # DockerHubPushAarch64: + # # needs: CheckLabels + # runs-on: [self-hosted, test-only] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + # DockerHubPushAmd64: + # # needs: CheckLabels + # runs-on: [self-hosted, test-only] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix amd64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + # DockerHubPush: + # needs: [DockerHubPushAmd64, DockerHubPushAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed aarch64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }} + # - name: Download changed amd64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }} + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/changed_images.json + # StyleCheck: + # needs: DockerHubPush + # runs-on: [self-hosted, builder] + # # We need additional `&& ! cancelled()` to have the job being able to cancel + # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + # steps: + # - name: Set envs + # run: echo "hi" # run: | # cat >> "$GITHUB_ENV" << 'EOF' # TEMP_PATH=${{ runner.temp }}/style_check @@ -246,48 +246,48 @@ jobs: # ######################################################################################### # #################################### ORDINARY BUILDS #################################### # ######################################################################################### - BuilderDebRelease: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # for performance artifact - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # BuilderDebRelease: + # needs: [DockerHubPush, FastTest] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # BuilderBinRelease: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -329,277 +329,277 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_asan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebUBsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_ubsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebTsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_tsan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebMsan: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_msan - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebDebug: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_debug - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Apply sparse checkout for contrib # in order to check that it doesn't break build - run: | - rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' - git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' - "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' - du -hs "$GITHUB_WORKSPACE/contrib" ||: - find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -# ########################################################################################## -# ##################################### SPECIAL BUILDS ##################################### -# ########################################################################################## -# BuilderBinClangTidy: -# needs: [DockerHubPush, FastTest, StyleCheck] -# runs-on: [self-hosted, builder] -# steps: -# - name: Set envs -# run: | -# cat >> "$GITHUB_ENV" << 'EOF' -# TEMP_PATH=${{runner.temp}}/build_check -# IMAGES_PATH=${{runner.temp}}/images_path -# REPO_COPY=${{runner.temp}}/build_check/ClickHouse -# CACHES_PATH=${{runner.temp}}/../ccaches -# BUILD_NAME=binary_tidy -# EOF -# - name: Download changed images + # BuilderDebAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_asan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebUBsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_ubsan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebTsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_tsan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebMsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_msan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebDebug: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_debug + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Apply sparse checkout for contrib # in order to check that it doesn't break build + # run: | + # rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' + # git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' + # "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' + # du -hs "$GITHUB_WORKSPACE/contrib" ||: + # find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images # uses: actions/download-artifact@v3 # with: # name: changed_images @@ -627,47 +627,47 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwin: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwin: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderBinAarch64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -750,47 +750,47 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" # BuilderBinPPC64: # needs: [DockerHubPush, FastTest, StyleCheck] # runs-on: [self-hosted, builder] @@ -917,30 +917,30 @@ jobs: # ############################################################################################ # ##################################### Docker images ####################################### # ############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # DockerServerImages: + # needs: + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # ############################################################################################ # ##################################### BUILD REPORTER ####################################### # ############################################################################################ @@ -1104,40 +1104,40 @@ jobs: # ############################################################################################## # ########################### FUNCTIONAl STATELESS TESTS ####################################### # ############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestRelease: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (release) + # REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + # KILL_TIMEOUT=10800 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # FunctionalStatelessTestReleaseDatabaseReplicated0: # needs: [BuilderDebRelease] # runs-on: [self-hosted, func-tester] @@ -1818,112 +1818,112 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (aarch64) + # REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + # KILL_TIMEOUT=10800 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAsan0: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (asan) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=4 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAsan1: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (asan) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=4 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # FunctionalStatelessTestAsan2: # needs: [BuilderDebAsan] # runs-on: [self-hosted, func-tester] @@ -1996,114 +1996,114 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan0: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan1: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan2: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # FunctionalStatelessTestTsan3: # needs: [BuilderDebTsan] # runs-on: [self-hosted, func-tester] @@ -2176,45 +2176,9 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan0: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestUBsan1: +# FunctionalStatelessTestUBsan0: # needs: [BuilderDebUBsan] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | @@ -2224,7 +2188,7 @@ jobs: # CHECK_NAME=Stateless tests (ubsan) # REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_NUM=0 # RUN_BY_HASH_TOTAL=2 # EOF # - name: Download json reports @@ -2248,117 +2212,45 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan3: +# # FunctionalStatelessTestUBsan1: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (ubsan) +# # REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=2 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: # needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | @@ -2368,7 +2260,7 @@ jobs: # CHECK_NAME=Stateless tests (msan) # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_NUM=0 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -2392,9 +2284,9 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan4: +# FunctionalStatelessTestMsan1: # needs: [BuilderDebMsan] -# runs-on: [self-hosted, func-tester] +# runs-on: [self-hosted, builder] # steps: # - name: Set envs # run: | @@ -2404,7 +2296,7 @@ jobs: # CHECK_NAME=Stateless tests (msan) # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_NUM=1 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -2428,7 +2320,43 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" -# FunctionalStatelessTestMsan5: +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: # needs: [BuilderDebMsan] # runs-on: [self-hosted, func-tester] # steps: @@ -2440,7 +2368,7 @@ jobs: # CHECK_NAME=Stateless tests (msan) # REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse # KILL_TIMEOUT=10800 -# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_NUM=3 # RUN_BY_HASH_TOTAL=6 # EOF # - name: Download json reports @@ -2464,114 +2392,186 @@ jobs: # docker ps --quiet | xargs --no-run-if-empty docker kill ||: # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: # sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug0: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug1: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug2: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # FunctionalStatelessTestDebug3: # needs: [BuilderDebDebug] # runs-on: [self-hosted, func-tester] @@ -2726,244 +2726,244 @@ jobs: # ############################################################################################## # ############################ FUNCTIONAl STATEFUL TESTS ####################################### # ############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestRelease: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (release) + # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (aarch64) + # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestAsan: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (asan) + # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestTsan: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (tsan) + # REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestMsan: + # needs: [BuilderDebMsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_msan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (msan) + # REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestUBsan: + # needs: [BuilderDebUBsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_ubsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (ubsan) + # REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestDebug: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (debug) + # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # # Parallel replicas # FunctionalStatefulTestDebugParallelReplicas: # needs: [BuilderDebDebug] @@ -3654,111 +3654,111 @@ jobs: # ############################################################################################# # ############################# INTEGRATION TESTS ############################################# # ############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" + # IntegrationTestsAsan0: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # IntegrationTestsAsan1: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # IntegrationTestsAsan2: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" # IntegrationTestsAsan3: # needs: [BuilderDebAsan] # runs-on: [self-hosted, stress-tester] diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index dadb484c449e..172b4128dfc9 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -307,14 +307,14 @@ # python3 mark_release_ready.py -name: docker-check +# name: docker-check -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 -on: # yamllint disable-line rule:truthy - push: +# on: # yamllint disable-line rule:truthy +# push: # pull_request: # types: # - synchronize @@ -423,7 +423,7 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - + # StyleCheck: # needs: DockerHubPush # runs-on: [self-hosted, style-checker] diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go index 0a849a9f520a..a5770e47582c 100644 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd" ) func main() { diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go index 503d8e41fb7a..95aacb6c994f 100644 --- a/programs/diagnostics/cmd/collect.go +++ b/programs/diagnostics/cmd/collect.go @@ -5,15 +5,15 @@ import ( "os" "strings" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" + _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors/system" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs" + _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs/file" + _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs/terminal" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/utils" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go index 750576dda252..ff88aafd858e 100644 --- a/programs/diagnostics/cmd/help.go +++ b/programs/diagnostics/cmd/help.go @@ -4,10 +4,10 @@ import ( "fmt" "os" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go index 7671506ba596..c9d41a76151f 100644 --- a/programs/diagnostics/cmd/params/params_test.go +++ b/programs/diagnostics/cmd/params/params_test.go @@ -5,8 +5,8 @@ import ( "sort" "testing" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go index 92368bce6f3e..fcddefaa9ad8 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ b/programs/diagnostics/internal/collectors/clickhouse/config.go @@ -4,11 +4,11 @@ import ( "fmt" "path/filepath" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/data" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/utils" "github.com/pkg/errors" ) diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go index b4435b62ea2f..bddf371f4da1 100644 --- a/programs/diagnostics/internal/platform/manager.go +++ b/programs/diagnostics/internal/platform/manager.go @@ -4,8 +4,8 @@ import ( "errors" "sync" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" - "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/data" + "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/database" ) var once sync.Once From 6bbc3c399619466bf68bcfc07a1e4a83a88af1ef Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 12:23:45 -0400 Subject: [PATCH 182/213] CHECK FOR only binary image --- .github/workflows/binary_image.yml | 164 ++++++++++++++--------------- .github/workflows/pull_tests.yml | 12 +-- 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 51e40e922996..ecab484e7af6 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -1,84 +1,84 @@ -name: binary-image +# name: binary-image -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 -on: # yamllint disable-line rule:truthy - push: - # pull_request: - # types: - # - synchronize - # - reopened - # - opened - # branches: - # - master - # paths-ignore: - # - 'CHANGELOG.md' - # - 'README.md' - # - 'SECURITY.md' - # - 'docker/docs/**' - # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' -jobs: - DockerHubPushAarch64: - runs-on: [self-hosted, builder] - steps: - - name: test - run: env - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json \ No newline at end of file +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# jobs: +# DockerHubPushAarch64: +# runs-on: [self-hosted, builder] +# steps: +# - name: test +# run: env +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json +# DockerHubPushAmd64: +# runs-on: [self-hosted, builder] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix amd64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json +# DockerHubPush: +# needs: [DockerHubPushAmd64, DockerHubPushAarch64] +# runs-on: [self-hosted, builder] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed aarch64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }} +# - name: Download changed amd64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }} +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/changed_images.json \ No newline at end of file diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml index 172b4128dfc9..5e697371b2b1 100644 --- a/.github/workflows/pull_tests.yml +++ b/.github/workflows/pull_tests.yml @@ -307,14 +307,14 @@ # python3 mark_release_ready.py -# name: docker-check +name: docker-check -# env: -# # Force the stdout and stderr streams to be unbuffered -# PYTHONUNBUFFERED: 1 +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 -# on: # yamllint disable-line rule:truthy -# push: +on: # yamllint disable-line rule:truthy + push: # pull_request: # types: # - synchronize From 187ea4bd9393c1adec75c7afb9cd4fc6999f1fd2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 13:58:07 -0400 Subject: [PATCH 183/213] CHECK FOR only binary image --- programs/diagnostics/cmd/collect.go | 32 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go index 95aacb6c994f..daf5537abb09 100644 --- a/programs/diagnostics/cmd/collect.go +++ b/programs/diagnostics/cmd/collect.go @@ -1,25 +1,39 @@ package cmd + import ( "fmt" "os" "strings" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" - _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors/system" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs" - _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs/file" - _ "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs/terminal" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/utils" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" ) +func main() { + githubRepository := os.Getenv("GITHUB_REPOSITORY") + // if githubRepository == "" { + // log.Fatal("GITHUB_REPOSITORY environment variable is not set") + + imports := fmt.Sprintf(` + "github.com/%s/programs/diagnostics/cmd/params" + "github.com/%s/programs/diagnostics/internal" + "github.com/%s/programs/diagnostics/internal/collectors" + _ "github.com/%s/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/%s/programs/diagnostics/internal/collectors/system" + "github.com/%s/programs/diagnostics/internal/outputs" + _ "github.com/%s/programs/diagnostics/internal/outputs/file" + _ "github.com/%s/programs/diagnostics/internal/outputs/terminal" + "github.com/%s/programs/diagnostics/internal/platform/utils" + `, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository) + + // Rest of your code +} + + + var id string var output = params.StringOptionsVar{ Options: outputs.GetOutputNames(), From 5d485be3d56d5af45667dc809ae2e907bda9c481 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 14:26:12 -0400 Subject: [PATCH 184/213] CHECK FOR only binary image --- .../cmd/clickhouse-diagnostics/main.go | 4 ++++ programs/diagnostics/cmd/collect.go | 1 - programs/diagnostics/cmd/help.go | 18 ++++++++++++++---- .../diagnostics/cmd/params/params_test.go | 14 ++++++++++++-- .../internal/collectors/clickhouse/config.go | 19 ++++++++++++++----- .../diagnostics/internal/platform/manager.go | 13 +++++++++++-- 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go index a5770e47582c..bdc62235d82f 100644 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go @@ -1,5 +1,9 @@ package main +func GetGITHUBREPOSITORY() string { + return GetEnv("GITHUB_REPOSITORY") +} + import ( "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd" ) diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go index daf5537abb09..250f8f41b329 100644 --- a/programs/diagnostics/cmd/collect.go +++ b/programs/diagnostics/cmd/collect.go @@ -29,7 +29,6 @@ func main() { "github.com/%s/programs/diagnostics/internal/platform/utils" `, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository) - // Rest of your code } diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go index ff88aafd858e..efa665311cfd 100644 --- a/programs/diagnostics/cmd/help.go +++ b/programs/diagnostics/cmd/help.go @@ -4,14 +4,24 @@ import ( "fmt" "os" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/outputs" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) +func main() { + githubRepository := os.Getenv("GITHUB_REPOSITORY") + // if githubRepository == "" { + // log.Fatal("GITHUB_REPOSITORY environment variable is not set") + + imports := fmt.Sprintf(` + "github.com/%s/programs/diagnostics/cmd/params" + "github.com/%s/programs/diagnostics/internal/collectors" + "github.com/%s/programs/diagnostics/internal/outputs" + "github.com/%s/programs/diagnostics/internal/platform/config" + `, githubRepository, githubRepository, githubRepository, githubRepository) + +} + var cHelp = params.StringOptionsVar{ Options: collectors.GetCollectorNames(false), Value: "", diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go index c9d41a76151f..c692d5091997 100644 --- a/programs/diagnostics/cmd/params/params_test.go +++ b/programs/diagnostics/cmd/params/params_test.go @@ -5,12 +5,22 @@ import ( "sort" "testing" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd/params" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) +func main() { + githubRepository := os.Getenv("GITHUB_REPOSITORY") + // if githubRepository == "" { + // log.Fatal("GITHUB_REPOSITORY environment variable is not set") + + imports := fmt.Sprintf(` + "github.com/%s/programs/diagnostics/cmd/params" + "github.com/%s/programs/diagnostics/internal/platform/config" + `, githubRepository, githubRepository) + +} + var conf = map[string]config.Configuration{ "config": { Params: []config.ConfigParam{ diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go index fcddefaa9ad8..ca96596b867f 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ b/programs/diagnostics/internal/collectors/clickhouse/config.go @@ -4,14 +4,23 @@ import ( "fmt" "path/filepath" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/collectors" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/config" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/data" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/utils" "github.com/pkg/errors" ) +githubRepository := os.Getenv("GITHUB_REPOSITORY") +// if githubRepository == "" { +// log.Fatal("GITHUB_REPOSITORY environment variable is not set") + +imports := fmt.Sprintf(` + "github.com/%s/programs/diagnostics/internal/collectors" + "github.com/%s/programs/diagnostics/internal/platform" + "github.com/%s/programs/diagnostics/internal/platform/config" + "github.com/%s/programs/diagnostics/internal/platform/data" + "github.com/%s/programs/diagnostics/internal/platform/utils" +`, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository) + +} + type ConfigCollector struct { resourceManager *platform.ResourceManager } diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go index bddf371f4da1..3b556b65c583 100644 --- a/programs/diagnostics/internal/platform/manager.go +++ b/programs/diagnostics/internal/platform/manager.go @@ -4,10 +4,19 @@ import ( "errors" "sync" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/data" - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/internal/platform/database" ) +func main() { + githubRepository := os.Getenv("GITHUB_REPOSITORY") + // if githubRepository == "" { + // log.Fatal("GITHUB_REPOSITORY environment variable is not set") + + imports := fmt.Sprintf(` + "github.com/%s/programs/diagnostics/internal/platform/data" + "github.com/%s/programs/diagnostics/internal/platform/database" + `, githubRepository, githubRepository) + +} var once sync.Once var dbInit sync.Once From 58a1291e607d88f1d4a499353a0a94477e31798b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 15:55:14 -0400 Subject: [PATCH 185/213] CHECK FOR only binary image --- docker/packager/binary/Dockerfile | 3 -- docker/packager/binary/build.sh | 4 +-- programs/diagnostics/Makefile | 2 +- .../cmd/clickhouse-diagnostics/main.go | 6 +--- programs/diagnostics/cmd/collect.go | 31 ++++++------------- programs/diagnostics/cmd/help.go | 18 +++-------- .../diagnostics/cmd/params/params_test.go | 14 ++------- .../internal/collectors/clickhouse/config.go | 19 +++--------- .../diagnostics/internal/platform/manager.go | 13 ++------ 9 files changed, 25 insertions(+), 85 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index be81bb214d0f..450227b5027d 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -111,9 +111,6 @@ COPY --from=cctools /cctools /cctools RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir -ARG GITHUB_REPOSITORY - -ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 28c584f984df..c0803c741475 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -x -e -github_repository=${GITHUB_REPOSITORY} - exec &> >(ts) ccache_status () { @@ -41,7 +39,7 @@ if [ -n "$MAKE_DEB" ]; then # We need to check if clickhouse-diagnostics is fine and build it ( cd /build/programs/diagnostics - GITHUB_REPOSITORY="${github_repository}" make test-no-docker + make test-no-docker GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build mv clickhouse-diagnostics .. ) diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 1c5372de07a3..2e85002b871d 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -5,7 +5,7 @@ BUILD_DIR=dist TIMESTAMP := $(shell date +%Y%m%d-%H%M) COMMIT := $(shell git rev-parse --short HEAD) -MODULE := github.com/${GITHUB_REPOSITORY}/programs/diagnostics +MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics VERSION := v.dev-${TIMESTAMP} DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" diff --git a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go index bdc62235d82f..0a849a9f520a 100644 --- a/programs/diagnostics/cmd/clickhouse-diagnostics/main.go +++ b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go @@ -1,11 +1,7 @@ package main -func GetGITHUBREPOSITORY() string { - return GetEnv("GITHUB_REPOSITORY") -} - import ( - "github.com/${GITHUB_REPOSITORY}/programs/diagnostics/cmd" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" ) func main() { diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go index 250f8f41b329..503d8e41fb7a 100644 --- a/programs/diagnostics/cmd/collect.go +++ b/programs/diagnostics/cmd/collect.go @@ -1,38 +1,25 @@ package cmd - import ( "fmt" "os" "strings" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" ) -func main() { - githubRepository := os.Getenv("GITHUB_REPOSITORY") - // if githubRepository == "" { - // log.Fatal("GITHUB_REPOSITORY environment variable is not set") - - imports := fmt.Sprintf(` - "github.com/%s/programs/diagnostics/cmd/params" - "github.com/%s/programs/diagnostics/internal" - "github.com/%s/programs/diagnostics/internal/collectors" - _ "github.com/%s/programs/diagnostics/internal/collectors/clickhouse" - _ "github.com/%s/programs/diagnostics/internal/collectors/system" - "github.com/%s/programs/diagnostics/internal/outputs" - _ "github.com/%s/programs/diagnostics/internal/outputs/file" - _ "github.com/%s/programs/diagnostics/internal/outputs/terminal" - "github.com/%s/programs/diagnostics/internal/platform/utils" - `, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository) - -} - - - var id string var output = params.StringOptionsVar{ Options: outputs.GetOutputNames(), diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go index efa665311cfd..750576dda252 100644 --- a/programs/diagnostics/cmd/help.go +++ b/programs/diagnostics/cmd/help.go @@ -4,24 +4,14 @@ import ( "fmt" "os" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) -func main() { - githubRepository := os.Getenv("GITHUB_REPOSITORY") - // if githubRepository == "" { - // log.Fatal("GITHUB_REPOSITORY environment variable is not set") - - imports := fmt.Sprintf(` - "github.com/%s/programs/diagnostics/cmd/params" - "github.com/%s/programs/diagnostics/internal/collectors" - "github.com/%s/programs/diagnostics/internal/outputs" - "github.com/%s/programs/diagnostics/internal/platform/config" - `, githubRepository, githubRepository, githubRepository, githubRepository) - -} - var cHelp = params.StringOptionsVar{ Options: collectors.GetCollectorNames(false), Value: "", diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go index c692d5091997..7671506ba596 100644 --- a/programs/diagnostics/cmd/params/params_test.go +++ b/programs/diagnostics/cmd/params/params_test.go @@ -5,22 +5,12 @@ import ( "sort" "testing" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) -func main() { - githubRepository := os.Getenv("GITHUB_REPOSITORY") - // if githubRepository == "" { - // log.Fatal("GITHUB_REPOSITORY environment variable is not set") - - imports := fmt.Sprintf(` - "github.com/%s/programs/diagnostics/cmd/params" - "github.com/%s/programs/diagnostics/internal/platform/config" - `, githubRepository, githubRepository) - -} - var conf = map[string]config.Configuration{ "config": { Params: []config.ConfigParam{ diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go index ca96596b867f..92368bce6f3e 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ b/programs/diagnostics/internal/collectors/clickhouse/config.go @@ -4,23 +4,14 @@ import ( "fmt" "path/filepath" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" "github.com/pkg/errors" ) -githubRepository := os.Getenv("GITHUB_REPOSITORY") -// if githubRepository == "" { -// log.Fatal("GITHUB_REPOSITORY environment variable is not set") - -imports := fmt.Sprintf(` - "github.com/%s/programs/diagnostics/internal/collectors" - "github.com/%s/programs/diagnostics/internal/platform" - "github.com/%s/programs/diagnostics/internal/platform/config" - "github.com/%s/programs/diagnostics/internal/platform/data" - "github.com/%s/programs/diagnostics/internal/platform/utils" -`, githubRepository, githubRepository, githubRepository, githubRepository, githubRepository) - -} - type ConfigCollector struct { resourceManager *platform.ResourceManager } diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go index 3b556b65c583..b4435b62ea2f 100644 --- a/programs/diagnostics/internal/platform/manager.go +++ b/programs/diagnostics/internal/platform/manager.go @@ -4,19 +4,10 @@ import ( "errors" "sync" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" ) -func main() { - githubRepository := os.Getenv("GITHUB_REPOSITORY") - // if githubRepository == "" { - // log.Fatal("GITHUB_REPOSITORY environment variable is not set") - - imports := fmt.Sprintf(` - "github.com/%s/programs/diagnostics/internal/platform/data" - "github.com/%s/programs/diagnostics/internal/platform/database" - `, githubRepository, githubRepository) - -} var once sync.Once var dbInit sync.Once From b5c7bfe777ffca18a4a014f8659ce3e5c0101bbe Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 15:58:23 -0400 Subject: [PATCH 186/213] CHECK FOR only binary image --- docker/packager/binary/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 450227b5027d..ae8309953905 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -111,6 +111,5 @@ COPY --from=cctools /cctools /cctools RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir - COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] From 757d5fcb2484c58d7894c582ba4174dba1d32d08 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 16:07:03 -0400 Subject: [PATCH 187/213] CHECK FOR only binary image --- docker/packager/binary/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index ae8309953905..450227b5027d 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -111,5 +111,6 @@ COPY --from=cctools /cctools /cctools RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir + COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] From a38f3f9657f0b120fa0d8ce9cca0032e24ef5959 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 20 Jun 2023 17:01:10 -0400 Subject: [PATCH 188/213] CHECK FOR only binary image --- .github/workflows/binary_image.yml | 162 +++++++++++++-------------- .github/workflows/docker_server.yaml | 41 ------- docker/packager/binary/Dockerfile | 1 + 3 files changed, 81 insertions(+), 123 deletions(-) delete mode 100644 .github/workflows/docker_server.yaml diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index ecab484e7af6..bd94d5ee78f8 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -1,84 +1,82 @@ -# name: binary-image +name: binary-image -# env: -# # Force the stdout and stderr streams to be unbuffered -# PYTHONUNBUFFERED: 1 +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 -# on: # yamllint disable-line rule:truthy -# push: -# # pull_request: -# # types: -# # - synchronize -# # - reopened -# # - opened -# # branches: -# # - master -# # paths-ignore: -# # - 'CHANGELOG.md' -# # - 'README.md' -# # - 'SECURITY.md' -# # - 'docker/docs/**' -# # - 'docs/**' -# # - 'utils/check-style/aspell-ignore/**' -# jobs: -# DockerHubPushAarch64: -# runs-on: [self-hosted, builder] -# steps: -# - name: test -# run: env -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Images check -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_images_check.py --suffix aarch64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images_aarch64 -# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json -# DockerHubPushAmd64: -# runs-on: [self-hosted, builder] -# steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Images check -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_images_check.py --suffix amd64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images_amd64 -# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json -# DockerHubPush: -# needs: [DockerHubPushAmd64, DockerHubPushAarch64] -# runs-on: [self-hosted, builder] -# steps: -# - name: Check out repository code -# uses: ClickHouse/checkout@v1 -# with: -# clear-repository: true -# - name: Download changed aarch64 images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images_aarch64 -# path: ${{ runner.temp }} -# - name: Download changed amd64 images -# uses: actions/download-artifact@v3 -# with: -# name: changed_images_amd64 -# path: ${{ runner.temp }} -# - name: Images check -# run: | -# cd "$GITHUB_WORKSPACE/tests/ci" -# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 -# - name: Upload images files to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: changed_images -# path: ${{ runner.temp }}/changed_images.json \ No newline at end of file +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json \ No newline at end of file diff --git a/.github/workflows/docker_server.yaml b/.github/workflows/docker_server.yaml deleted file mode 100644 index eedc6e9ca656..000000000000 --- a/.github/workflows/docker_server.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: checkdocker - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - -on: # yamllint disable-line rule:truthy - push: -########################################################################################## -##################################### SMALL CHECKS ####################################### -########################################################################################## -jobs: - DockerServerImages: - runs-on: [self-hosted, docker] - steps: - - name: Check value - run: | - DOCKER_REPO="${DOCKER_REPO:-docker.io}" - echo "DOCKER_REPO value: $DOCKER_REPO" - - - - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - # - name: Check docker clickhouse/clickhouse-server building - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # DOCKER_REPO="${DOCKER_REPO:-docker.io}" - # python3 docker_server.py --release-type head --no-push \ - # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server - # python3 docker_server.py --release-type head --no-push \ - # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper - # - name: Cleanup - # if: always() - # run: | - # docker ps --quiet | xargs --no-run-if-empty docker kill ||: - # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - # sudo rm -fr "$TEMP_PATH" \ No newline at end of file diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 450227b5027d..98bdcb9f87a3 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -112,5 +112,6 @@ RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir + COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] From c9bb284d6aae7d8965e2a553fb75c0248f15ee7d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 13:55:15 -0400 Subject: [PATCH 189/213] testing for arch --- .github/workflows/binary_image.yml | 116 +++++++++++++++-------------- 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index bd94d5ee78f8..d55c6142dac3 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,61 +22,63 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, builder] + runs-on: [self-hosted, arm64-silicon] steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json \ No newline at end of file + - name: env + run: env + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + # DockerHubPushAmd64: + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix amd64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + # DockerHubPush: + # needs: [DockerHubPushAmd64, DockerHubPushAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed aarch64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }} + # - name: Download changed amd64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }} + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/changed_images.json \ No newline at end of file From f10ed5c0a1ccd1889712c613c8d95c54d2ea68d0 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 14:12:13 -0400 Subject: [PATCH 190/213] testing for arch64-runner --- .github/workflows/binary_image.yml | 103 +++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index d55c6142dac3..19c46ba090c3 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -26,19 +26,19 @@ jobs: steps: - name: env run: env - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_images_check.py --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json # DockerHubPushAmd64: # runs-on: [self-hosted, builder] # steps: @@ -81,4 +81,79 @@ jobs: # uses: actions/upload-artifact@v3 # with: # name: changed_images - # path: ${{ runner.temp }}/changed_images.json \ No newline at end of file + # path: ${{ runner.temp }}/changed_images.json + BuilderDebAarch64: + needs: [DockerHubPush] + runs-on: [self-hosted, arm64-silicon] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + submodules: true + fetch-depth: 0 # for performance artifact + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + InstallPackagesTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, arm64-silicon] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/test_install + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Install packages (arm64) + REPO_COPY=${{runner.temp}}/test_install/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Test packages installation + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 install_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" \ No newline at end of file From 23c8bacc5e9e2efe1fcc2d005d1a0de1216a799a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 14:14:18 -0400 Subject: [PATCH 191/213] testing for arch64 --- .github/workflows/binary_image.yml | 88 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 19c46ba090c3..d04b01612e13 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -24,8 +24,6 @@ jobs: DockerHubPushAarch64: runs-on: [self-hosted, arm64-silicon] steps: - - name: env - run: env - name: Check out repository code uses: ClickHouse/checkout@v1 with: @@ -39,49 +37,49 @@ jobs: with: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - # DockerHubPushAmd64: - # runs-on: [self-hosted, builder] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_images_check.py --suffix amd64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images_amd64 - # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - # DockerHubPush: - # needs: [DockerHubPushAmd64, DockerHubPushAarch64] - # runs-on: [self-hosted, builder] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download changed aarch64 images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }} - # - name: Download changed amd64 images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images_amd64 - # path: ${{ runner.temp }} - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images - # path: ${{ runner.temp }}/changed_images.json + DockerHubPushAmd64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json BuilderDebAarch64: needs: [DockerHubPush] runs-on: [self-hosted, arm64-silicon] From 2fe85f7404a3cc548f3d1eedf233cfc27ce66a8f Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 15:44:17 -0400 Subject: [PATCH 192/213] testing for containers --- tests/ci/docker_images_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 1c8d635d669c..c55c3690eb43 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,6 +410,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg + docker ps f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", From 2ad0ad362c9643d866067a363291d08bce52e5cd Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 15:46:45 -0400 Subject: [PATCH 193/213] undo --- tests/ci/docker_images_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c55c3690eb43..1c8d635d669c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,7 +410,6 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - docker ps f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", From 63aa27cdc589109911ed7b0a8435f11282e588e7 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 15:47:17 -0400 Subject: [PATCH 194/213] testing for containers --- tests/ci/docker_images_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 1c8d635d669c..eacb761dda46 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,6 +410,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg + docker ps, f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", From 5e5f6fdbe02b46a48651fc92db02042ffc702260 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Fri, 23 Jun 2023 16:08:45 -0400 Subject: [PATCH 195/213] undo --- tests/ci/docker_images_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index eacb761dda46..1c8d635d669c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,7 +410,6 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - docker ps, f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", From e86e49c6b476f11818588120358abc840c4f14ca Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 24 Jun 2023 23:12:05 -0400 Subject: [PATCH 196/213] added sudo for testing --- tests/ci/docker_images_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 1c8d635d669c..078739f2abdf 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,7 +410,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + f"sudo docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, From d909459df175298317bc301f62e9e7a39043430d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sat, 24 Jun 2023 23:33:09 -0400 Subject: [PATCH 197/213] updated runner in arch package --- .github/workflows/binary_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index d04b01612e13..0cba1ec5282a 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, arm64-silicon] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From 9d92b1d6fa162af7a1124c24dc5abc361f8cd8e8 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 25 Jun 2023 16:39:14 -0400 Subject: [PATCH 198/213] testing for arch runner in other job --- .github/workflows/binary_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 0cba1ec5282a..d04b01612e13 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, builder] + runs-on: [self-hosted, arm64-silicon] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From daa99629ad3d9e70a94100c233e7b6ea87d53c79 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 25 Jun 2023 18:17:46 -0400 Subject: [PATCH 199/213] updated docker run with sudo --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 5c464b70f7d9..7a91a04a0138 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -102,7 +102,7 @@ def run_docker_image_with_env( ccache_mount = "" cmd = ( - f"docker run --network=host --user={user} --rm {ccache_mount}" + f"sudo docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"--volume=/home/ubuntu/.aws/credentials:/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" From 0a68d21e9085aa535199aa232343dcf41a84130c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 25 Jun 2023 20:45:07 -0400 Subject: [PATCH 200/213] updated visudo to let ubuntu access without sudo --- .github/workflows/binary_image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index d04b01612e13..be4adb1feabd 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -19,7 +19,8 @@ on: # yamllint disable-line rule:truthy # - 'SECURITY.md' # - 'docker/docs/**' # - 'docs/**' - # - 'utils/check-style/aspell-ignore/**' + # - 'utils/check-style/aspell-ignore/**' + jobs: DockerHubPushAarch64: runs-on: [self-hosted, arm64-silicon] From 777aee5b51d3678a42307095774859591d7bd21c Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 25 Jun 2023 21:02:45 -0400 Subject: [PATCH 201/213] updated visudo to let ubuntu access without sudo --- .github/workflows/binary_image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index be4adb1feabd..1c710e97a297 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -20,7 +20,6 @@ on: # yamllint disable-line rule:truthy # - 'docker/docs/**' # - 'docs/**' # - 'utils/check-style/aspell-ignore/**' - jobs: DockerHubPushAarch64: runs-on: [self-hosted, arm64-silicon] From f53de881dc9b8892793680a04aa6ce0f88155568 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Sun, 25 Jun 2023 21:34:45 -0400 Subject: [PATCH 202/213] updated visudo to let ubuntu access without sudo --- .github/workflows/binary_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 1c710e97a297..be4adb1feabd 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -20,6 +20,7 @@ on: # yamllint disable-line rule:truthy # - 'docker/docs/**' # - 'docs/**' # - 'utils/check-style/aspell-ignore/**' + jobs: DockerHubPushAarch64: runs-on: [self-hosted, arm64-silicon] From 182957af4bdb0b896849cac02be848703ceb14fa Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 26 Jun 2023 07:45:29 -0400 Subject: [PATCH 203/213] checking without sudo for docker_images_check.py --- tests/ci/docker_images_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 078739f2abdf..1c8d635d669c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -410,7 +410,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - f"sudo docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, From 6603c8ca8a46aa197f74f7a69459a0eb89d46683 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 26 Jun 2023 07:55:00 -0400 Subject: [PATCH 204/213] testing for run --- .github/workflows/binary_image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index be4adb1feabd..1c710e97a297 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -20,7 +20,6 @@ on: # yamllint disable-line rule:truthy # - 'docker/docs/**' # - 'docs/**' # - 'utils/check-style/aspell-ignore/**' - jobs: DockerHubPushAarch64: runs-on: [self-hosted, arm64-silicon] From 159ea1d4d1ad8fec3773ed87a7467a057a5b3ae2 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 26 Jun 2023 08:00:47 -0400 Subject: [PATCH 205/213] testing for docker run without sudo --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 7a91a04a0138..5c464b70f7d9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -102,7 +102,7 @@ def run_docker_image_with_env( ccache_mount = "" cmd = ( - f"sudo docker run --network=host --user={user} --rm {ccache_mount}" + f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"--volume=/home/ubuntu/.aws/credentials:/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" From 2af7714f932fdf8f5eb3260afdcaff26b67d0dcc Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Mon, 26 Jun 2023 08:39:06 -0400 Subject: [PATCH 206/213] testing for arch runnner for install packages --- .github/workflows/binary_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 1c710e97a297..25a305cfc8e7 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -82,7 +82,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, arm64-silicon] + runs-on: [self-hosted, vm2] steps: - name: Set envs run: | From 701ea5b788453b831c34cf2f6111be988566eb1a Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 01:59:13 -0400 Subject: [PATCH 207/213] testing for arch runner on arm64vm --- .github/workflows/binary_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 25a305cfc8e7..d2726dadeb20 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, arm64-silicon] + runs-on: [self-hosted, test41] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -82,7 +82,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, vm2] + runs-on: [self-hosted, test41] steps: - name: Set envs run: | @@ -124,7 +124,7 @@ jobs: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" InstallPackagesTestAarch64: needs: [BuilderDebAarch64] - runs-on: [self-hosted, arm64-silicon] + runs-on: [self-hosted, test41] steps: - name: Set envs run: | From 562b96fbff2e7ded670b5e41b7531cc0873d6849 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 09:21:07 -0400 Subject: [PATCH 208/213] testing for arch runner on arm64vm for install job --- .github/workflows/binary_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index d2726dadeb20..22c00e378880 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, test41] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -55,7 +55,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, builder] + runs-on: [self-hosted, vm2] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -82,7 +82,7 @@ jobs: path: ${{ runner.temp }}/changed_images.json BuilderDebAarch64: needs: [DockerHubPush] - runs-on: [self-hosted, test41] + runs-on: [self-hosted, vm2] steps: - name: Set envs run: | From 48cac458678e8ff0463a1af2ca8f548727dd4a5b Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 10:26:52 -0400 Subject: [PATCH 209/213] check for rnv --- .github/workflows/binary_image.yml | 264 +++++++++++++++-------------- 1 file changed, 133 insertions(+), 131 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 22c00e378880..25ce940c98eb 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,136 +22,138 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - DockerHubPushAmd64: - runs-on: [self-hosted, builder] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_images_check.py --suffix amd64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, vm2] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download changed aarch64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_aarch64 - path: ${{ runner.temp }} - - name: Download changed amd64 images - uses: actions/download-artifact@v3 - with: - name: changed_images_amd64 - path: ${{ runner.temp }} - - name: Images check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - - name: Upload images files to artifacts - uses: actions/upload-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/changed_images.json - BuilderDebAarch64: - needs: [DockerHubPush] - runs-on: [self-hosted, vm2] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - fetch-depth: 0 # for performance artifact - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] runs-on: [self-hosted, test41] steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" \ No newline at end of file + - name: env + run: env + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + # DockerHubPushAmd64: + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix amd64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + # DockerHubPush: + # needs: [DockerHubPushAmd64, DockerHubPushAarch64] + # runs-on: [self-hosted, vm2] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed aarch64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }} + # - name: Download changed amd64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }} + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/changed_images.json + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, vm2] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # InstallPackagesTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, test41] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/test_install + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Install packages (arm64) + # REPO_COPY=${{runner.temp}}/test_install/ClickHouse + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Test packages installation + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 install_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" \ No newline at end of file From 9118f0f717962f4e75b26513e4d77453f05c501d Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 10:44:50 -0400 Subject: [PATCH 210/213] check for arm64 images --- .github/workflows/binary_image.yml | 114 ++++++++++++++--------------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 25ce940c98eb..7ab276572a2f 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -24,64 +24,62 @@ jobs: DockerHubPushAarch64: runs-on: [self-hosted, test41] steps: - - name: env - run: env - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_images_check.py --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json - # DockerHubPushAmd64: - # runs-on: [self-hosted, builder] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_images_check.py --suffix amd64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images_amd64 - # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json - # DockerHubPush: - # needs: [DockerHubPushAmd64, DockerHubPushAarch64] - # runs-on: [self-hosted, vm2] - # steps: - # - name: Check out repository code - # uses: ClickHouse/checkout@v1 - # with: - # clear-repository: true - # - name: Download changed aarch64 images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images_aarch64 - # path: ${{ runner.temp }} - # - name: Download changed amd64 images - # uses: actions/download-artifact@v3 - # with: - # name: changed_images_amd64 - # path: ${{ runner.temp }} - # - name: Images check - # run: | - # cd "$GITHUB_WORKSPACE/tests/ci" - # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - # - name: Upload images files to artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: changed_images - # path: ${{ runner.temp }}/changed_images.json + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, vm2] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json # BuilderDebAarch64: # needs: [DockerHubPush] # runs-on: [self-hosted, vm2] From c35a5606d0feb458a972d59a47320f85238ca615 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 19:24:20 -0400 Subject: [PATCH 211/213] building util and base image --- docker/packager/binary/Dockerfile | 2 -- docker/test/base/Dockerfile | 1 + docker/test/fasttest/Dockerfile | 4 ---- docker/test/util/Dockerfile | 1 + 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 98bdcb9f87a3..ae8309953905 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -111,7 +111,5 @@ COPY --from=cctools /cctools /cctools RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir - - COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 1ad6f7ab8c07..70159f033990 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -4,6 +4,7 @@ ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG + RUN apt-get update \ && apt-get install \ lcov \ diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 92324df63d35..98d58fa9767c 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -4,10 +4,6 @@ ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG -ARG GITHUB_REPOSITORY - -ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY - RUN apt-get update \ && apt-get install \ brotli \ diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index a49278e960b3..b93add045110 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -8,6 +8,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list # 15.0.2 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=16 + RUN apt-get update \ && apt-get install \ apt-transport-https \ From 71a475ed6e3d3c3226be13f592e97526f3191e49 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 19:27:43 -0400 Subject: [PATCH 212/213] updated runner, build util and base --- .github/workflows/binary_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index 7ab276572a2f..a2a896011e08 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -22,7 +22,7 @@ on: # yamllint disable-line rule:truthy # - 'utils/check-style/aspell-ignore/**' jobs: DockerHubPushAarch64: - runs-on: [self-hosted, test41] + runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 From b4a63102f25c851f3cf7c79c8e7b7784104c5615 Mon Sep 17 00:00:00 2001 From: umang brahmbhatt Date: Tue, 27 Jun 2023 19:33:10 -0400 Subject: [PATCH 213/213] updated runner, build util and base --- .github/workflows/binary_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml index a2a896011e08..0c07f7426ab0 100644 --- a/.github/workflows/binary_image.yml +++ b/.github/workflows/binary_image.yml @@ -55,7 +55,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, vm2] + runs-on: [self-hosted, builder] steps: - name: Check out repository code uses: ClickHouse/checkout@v1