From 4ddf9d8a5c02c2247df6bbe1e25a843ada4ea8f8 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:38:23 +0300 Subject: [PATCH 01/10] adjust get projects for mixed repos --- .github/workflows/get-projects.yaml | 37 +++++++++++++++++------------ 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/get-projects.yaml b/.github/workflows/get-projects.yaml index 4aa1941a..1702cefe 100644 --- a/.github/workflows/get-projects.yaml +++ b/.github/workflows/get-projects.yaml @@ -57,15 +57,14 @@ jobs: outputs: projects: >- - ${{ steps.get_projects_multi.outputs.projects || - steps.get_projects_mono.outputs.projects || - steps.get_projects_bento.outputs.projects || + ${{ steps.get_projects_multi.outputs.projects || + steps.get_projects_mono.outputs.projects || + steps.get_projects_bento.outputs.projects || steps.get_projects_bento_multi.outputs.projects }} steps: - name: Checkout source uses: actions/checkout@v4 - - name: Get build number and image version run: | if [[ -z "${{ inputs.image_version }}" ]] @@ -87,23 +86,31 @@ jobs: - name: Check repo type id: repo_type run: | - if [[ -f './Dockerfile' ]] - then - echo 'Определили репозиторий как моносервисный' - echo "repo_type='mono'" >> $GITHUB_OUTPUT - elif [[ -f './bentofile.yaml' ]] - then - echo 'Определили репозиторий как бенто сервис' + # Проверка для bentofile.yaml + if [[ -f './bentofile.yaml' ]]; then + echo 'Определили репозиторий как бенто-сервисный' echo "repo_type='bento'" >> $GITHUB_OUTPUT - elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml - then + elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml; then echo 'Определили репозиторий как бенто-мультисервисный' echo "repo_type='bento-multi'" >> $GITHUB_OUTPUT - else + fi + + # Если bentofile.yaml найден в inputs.project_path_mask, пропускаем проверку Dockerfile + if find "${{ inputs.project_path_mask }}/" -type f -name 'bentofile.yaml' | grep -q '.'; then + echo 'Пропускаем проверку Dockerfile, так как найден bentofile.yaml' + exit 0 + fi + + # Проверка для Dockerfile + if [[ -f './Dockerfile' ]]; then + echo 'Определили репозиторий как моносервисный' + echo "repo_type='mono'" >> $GITHUB_OUTPUT + elif find . -type f -name 'Dockerfile' | grep -q -m 1 Dockerfile; then echo 'Определили репозиторий как мультисервисный' echo "repo_type='multi'" >> $GITHUB_OUTPUT fi + - name: Get projects (multi) id: get_projects_multi run: | @@ -192,7 +199,7 @@ jobs: run: | # Выходим, если репозиторий не бенто мультисервисный if [[ ${{ steps.repo_type.outputs.repo_type }} != 'bento-multi' ]]; then exit 0; fi - + projects=$(for p in $(find * -maxdepth 3 -path "${{ inputs.project_path_mask }}/bentofile.yaml") do project_path=$(echo $p | sed -E 's|(.*)/bentofile\.yaml$|\1|') From 043e735a069344ebf9d2e22cfff33e337c6f13fa Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:54:52 +0300 Subject: [PATCH 02/10] revert to previous version for docker-multi --- .github/workflows/get-projects.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/get-projects.yaml b/.github/workflows/get-projects.yaml index 1702cefe..f5aff2bc 100644 --- a/.github/workflows/get-projects.yaml +++ b/.github/workflows/get-projects.yaml @@ -105,7 +105,7 @@ jobs: if [[ -f './Dockerfile' ]]; then echo 'Определили репозиторий как моносервисный' echo "repo_type='mono'" >> $GITHUB_OUTPUT - elif find . -type f -name 'Dockerfile' | grep -q -m 1 Dockerfile; then + else echo 'Определили репозиторий как мультисервисный' echo "repo_type='multi'" >> $GITHUB_OUTPUT fi From 542a28d65eb9103293cd3c3ac807f277ea528da6 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Mon, 16 Dec 2024 18:31:02 +0300 Subject: [PATCH 03/10] fix multi docker build --- .github/workflows/build-and-test-harbor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-harbor.yaml b/.github/workflows/build-and-test-harbor.yaml index 9268d8a6..91184238 100644 --- a/.github/workflows/build-and-test-harbor.yaml +++ b/.github/workflows/build-and-test-harbor.yaml @@ -135,7 +135,7 @@ jobs: if: ${{ env.IMAGE_EXIST == 'false' }} uses: docker/build-push-action@v6 with: - context: . + context: ${{ matrix.project.project_path }} file: ${{ matrix.project.dockerfile }} build-args: | build_number=${{ matrix.project.build_number }} From 417a931b95cbe8adb470bd5b28ba2d972d8b4db0 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:36:14 +0300 Subject: [PATCH 04/10] create app in ml-prod --- .github/workflows/publish-to-ml-prod-harbor.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-ml-prod-harbor.yaml b/.github/workflows/publish-to-ml-prod-harbor.yaml index dfa53502..d715c7dd 100644 --- a/.github/workflows/publish-to-ml-prod-harbor.yaml +++ b/.github/workflows/publish-to-ml-prod-harbor.yaml @@ -88,7 +88,9 @@ jobs: then echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} уже существует' else - echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создайте его сначала в dev окружении!' + echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создадим новый' + cd deploy + ./helm_create.sh ${{ env.TEAM }} ${{ matrix.project.project_name }} fi; - name: Install yq From 805590275986796b18708266e8c0d7272a8f67ec Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:51:18 +0300 Subject: [PATCH 05/10] add ml_prod file for helm_create script --- .github/workflows/publish-to-ml-prod-harbor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-ml-prod-harbor.yaml b/.github/workflows/publish-to-ml-prod-harbor.yaml index d715c7dd..5e366ae8 100644 --- a/.github/workflows/publish-to-ml-prod-harbor.yaml +++ b/.github/workflows/publish-to-ml-prod-harbor.yaml @@ -90,7 +90,7 @@ jobs: else echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создадим новый' cd deploy - ./helm_create.sh ${{ env.TEAM }} ${{ matrix.project.project_name }} + ./helm_create.sh ${{ env.TEAM }} ${{ matrix.project.project_name }} ml_prod fi; - name: Install yq From 5fa9936efba3990664cd79b5431a078fef9a1339 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Thu, 27 Feb 2025 19:28:06 +0300 Subject: [PATCH 06/10] Update build-and-push-harbor-bentoml.yaml --- .github/workflows/build-and-push-harbor-bentoml.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-harbor-bentoml.yaml b/.github/workflows/build-and-push-harbor-bentoml.yaml index 843e34fc..a79cecb2 100644 --- a/.github/workflows/build-and-push-harbor-bentoml.yaml +++ b/.github/workflows/build-and-push-harbor-bentoml.yaml @@ -85,7 +85,7 @@ jobs: - name: Install requirements run: | cd ${{ matrix.project.project_path }} - pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple + pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --timeout 600 - name: Import model run: | cd ${{ matrix.project.project_path }} From 8217fea3a8e116d45966ce7ba05f4a3d4104de39 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Thu, 27 Feb 2025 19:34:07 +0300 Subject: [PATCH 07/10] Update build-and-push-harbor-bentoml.yaml --- .github/workflows/build-and-push-harbor-bentoml.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-push-harbor-bentoml.yaml b/.github/workflows/build-and-push-harbor-bentoml.yaml index a79cecb2..d09ac7a7 100644 --- a/.github/workflows/build-and-push-harbor-bentoml.yaml +++ b/.github/workflows/build-and-push-harbor-bentoml.yaml @@ -63,6 +63,11 @@ jobs: name: Bento build runs-on: ${{ inputs.runner_bento }} needs: [get_projects] + env: + HF_HUB_ETAG_TIMEOUT: 86400 + HF_HUB_DOWNLOAD_TIMEOUT: 86400 + HF_ENDPOINT: https://artifactory.yc.prod.infra.sravni.market/artifactory/api/huggingfaceml/huggingface.co_proxy + HF_TOKEN: ${{ secrets.ARTIFACTORY_SVC_TOKEN }} strategy: max-parallel: 1 From 13c0e089b3c9703a3117e835e7ecfe29c763ac78 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Tue, 4 Mar 2025 16:30:07 +0300 Subject: [PATCH 08/10] Update build-and-push-harbor-bentoml.yaml --- .github/workflows/build-and-push-harbor-bentoml.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push-harbor-bentoml.yaml b/.github/workflows/build-and-push-harbor-bentoml.yaml index d09ac7a7..ed6eac6d 100644 --- a/.github/workflows/build-and-push-harbor-bentoml.yaml +++ b/.github/workflows/build-and-push-harbor-bentoml.yaml @@ -83,10 +83,10 @@ jobs: registry: ${{ secrets.HARBOR_REGISTRY }} username: ${{ secrets.HARBOR_LOGIN }} password: ${{ secrets.HARBOR_PASSWORD }} - - uses: bentoml/setup-bentoml-action@v1 + - uses: sravni/.github/.github/actions/setup@bento-mixed with: python-version: '3.10' - bentoml-version: 'main' + bentoml-version: '1.3.14' - name: Install requirements run: | cd ${{ matrix.project.project_path }} @@ -95,7 +95,7 @@ jobs: run: | cd ${{ matrix.project.project_path }} python import_model.py - - uses: bentoml/build-bento-action@v1 + - uses: sravni/.github/.github/actions/builder@bento-mixed with: context: ${{ matrix.project.project_path }} id: bento From 6afc17c5a7a1b59d1d4c03aa665f51bf411c7d4d Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Tue, 4 Mar 2025 16:31:24 +0300 Subject: [PATCH 09/10] Create action.yml --- .github/actions/builder/action.yml | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/actions/builder/action.yml diff --git a/.github/actions/builder/action.yml b/.github/actions/builder/action.yml new file mode 100644 index 00000000..22cc4fce --- /dev/null +++ b/.github/actions/builder/action.yml @@ -0,0 +1,53 @@ +# https://help.github.com/en/articles/metadata-syntax-for-github-actions +name: Build Bento bundle with GitHub Action +description: Build Bento with GitHub Action +author: bentoml +branding: + icon: 'anchor' + color: 'blue' +inputs: + bentofile: + description: "Optional path to bentofile.yaml" + required: false + version: + description: "Optional version for this given Bento. By default this will be auto-generated" + required: false + context: + description: "Build's context is the set of files located in the specified PATH or URL" + required: false + default: '.' +outputs: + bento-tag: + description: 'Bento tag' + value: ${{ steps.build.outputs.bento-tag }} + bento-name: + description: 'Bento name' + value: ${{ steps.build.outputs.bento-name }} + bento-version: + description: 'Bento version' + value: ${{ steps.build.outputs.bento-version }} + metadata: + description: 'Build result metadata (equivalent of `bentoml get :`)' + value: ${{ steps.build.outputs.bento-metadata }} +runs: + using: 'composite' + steps: + - shell: bash + id: build + run: | + build_args=() + if [[ -n "${{ inputs.bentofile }}" ]]; then + build_args+=(--bentofile ${{ inputs.bentofile }}) + fi + if [[ -n "${{ inputs.version }}" ]]; then + build_args+=(--version ${{ inputs.version }}) + fi + bento_tag=$(bentoml build -o tag ${build_args[@]} ${{ inputs.context }}) + bento_tag=${bento_tag##__tag__:} + echo "bento-tag=$bento_tag" >> $GITHUB_OUTPUT + echo "bento-name=$(echo $bento_tag | cut -d: -f1)" >> $GITHUB_OUTPUT + echo "bento-version=$(echo $bento_tag | cut -d: -f2)" >> $GITHUB_OUTPUT + # Multiline string for gh action https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + echo "bento-metadata<> $GITHUB_OUTPUT + echo "$(bentoml get $bento_tag -o json)" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT From 1716cb2683f5eb8d4d79ec516a4ebd852d1c03f3 Mon Sep 17 00:00:00 2001 From: makar savchenko <64556050+MixwelInc@users.noreply.github.com> Date: Tue, 4 Mar 2025 16:31:57 +0300 Subject: [PATCH 10/10] Create action.yml --- .github/actions/setup/action.yml | 97 ++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..15c3d572 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,97 @@ +name: "Setup BentoML" +description: "Bootstrap and setup base environment for BentoML" +branding: + icon: 'box' + color: 'orange' +inputs: + bentoml-version: + description: The version of BentoML to use. Leave it empty to use the latest version, set to 'main' to use 'nightly' + required: false + default: "" + python-version: + description: Default Python version + required: false + default: "3.10" + architecture: + description: 'Which architecture to run on' + required: false + default: x64 +runs: + using: composite + steps: + - name: Checkout source + shell: bash + run: | + set -eux + echo "Checking out the repository..." + git --version + + - name: Configure APT proxy & install Python + shell: bash + run: | + set -eux + + UBUNTU_VERSION=$(grep "VERSION_CODENAME" /etc/os-release | cut -d'=' -f2) + ARTIFACTORY_URL="https://artifactory.yc.prod.infra.sravni.market/artifactory/ubuntu_proxy" + + echo "Configuring APT proxy for Artifactory ($UBUNTU_VERSION)..." + + echo "deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-backports main restricted universe multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security multiverse" | sudo tee /etc/apt/sources.list + + echo "Updating APT cache..." + sudo apt update -o Acquire::https::Verify-Peer=false -o Acquire::https::Verify-Host=false + + PYTHON_VERSION="${{ inputs.python-version }}" + + echo "Installing Python ${PYTHON_VERSION} from Artifactory..." + sudo apt install -y --no-install-recommends \ + python${PYTHON_VERSION} \ + ca-certificates + + if [[ ! -f "/usr/bin/python3" ]]; then + sudo ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 + fi + + echo "Python ${PYTHON_VERSION} successfully installed from Artifactory!" + + - name: Verify Python installation + shell: bash + run: | + set -eux + echo "Using system Python version: ${{ inputs.python-version }}" + python3 --version + + - name: Configure pip cache manually + shell: bash + run: | + set -eux + export PIP_CACHE_DIR="$HOME/.cache/pip" + mkdir -p "$PIP_CACHE_DIR" + echo "PIP cache directory: $PIP_CACHE_DIR" + echo "PIP_CACHE_DIR=$PIP_CACHE_DIR" >> "$GITHUB_ENV" + + - name: Install dependencies with caching + shell: bash + run: | + set -eux + + echo "Installing BentoML..." + if [[ -n "${{ inputs.bentoml-version }}" ]]; then + pip install -U "bentoml==${{ inputs.bentoml-version }}" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300 + else + pip install -U "bentoml>=1.0.25" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300 + fi + + echo "Successfully installed BentoML: $(bentoml --version)" + + echo "Installing hatch and towncrier..." + pip install hatch towncrier -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300