adjust get projects for bento+docker repos

.github/actions/builder/action.yml

@@ -0,0 +1,53 @@
+# https://help.github.com/en/articles/metadata-syntax-for-github-actions
+name: Build Bento bundle with GitHub Action
+description: Build Bento with GitHub Action
+author: bentoml
+branding:
+  icon: 'anchor'
+  color: 'blue'
+inputs:
+  bentofile:
+    description: "Optional path to bentofile.yaml"
+    required: false
+  version:
+    description: "Optional version for this given Bento. By default this will be auto-generated"
+    required: false
+  context:
+    description: "Build's context is the set of files located in the specified PATH or URL"
+    required: false
+    default: '.'
+outputs:
+  bento-tag:
+    description: 'Bento tag'
+    value: ${{ steps.build.outputs.bento-tag }}
+  bento-name:
+    description: 'Bento name'
+    value: ${{ steps.build.outputs.bento-name }}
+  bento-version:
+    description: 'Bento version'
+    value: ${{ steps.build.outputs.bento-version }}
+  metadata:
+    description: 'Build result metadata (equivalent of `bentoml get <bento_name>:<bento_version>`)'
+    value: ${{ steps.build.outputs.bento-metadata }}
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      id: build
+      run: |
+        build_args=()
+        if [[ -n "${{ inputs.bentofile }}" ]]; then
+          build_args+=(--bentofile ${{ inputs.bentofile }})
+        fi
+        if [[ -n "${{ inputs.version }}" ]]; then
+          build_args+=(--version ${{ inputs.version }})
+        fi
+        bento_tag=$(bentoml build -o tag ${build_args[@]} ${{ inputs.context }})
+        bento_tag=${bento_tag##__tag__:}
+        echo "bento-tag=$bento_tag" >> $GITHUB_OUTPUT
+        echo "bento-name=$(echo $bento_tag | cut -d: -f1)" >> $GITHUB_OUTPUT
+        echo "bento-version=$(echo $bento_tag | cut -d: -f2)" >> $GITHUB_OUTPUT
+        # Multiline string for gh action https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
+        echo "bento-metadata<<EOF" >> $GITHUB_OUTPUT
+        echo "$(bentoml get $bento_tag -o json)" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT

.github/actions/setup/action.yml

@@ -0,0 +1,97 @@
+name: "Setup BentoML"
+description: "Bootstrap and setup base environment for BentoML"
+branding:
+  icon: 'box'
+  color: 'orange'
+inputs:
+  bentoml-version:
+    description: The version of BentoML to use. Leave it empty to use the latest version, set to 'main' to use 'nightly'
+    required: false
+    default: ""
+  python-version:
+    description: Default Python version
+    required: false
+    default: "3.10"
+  architecture:
+    description: 'Which architecture to run on'
+    required: false
+    default: x64
+runs:
+  using: composite
+  steps:
+    - name: Checkout source
+      shell: bash
+      run: |
+        set -eux
+        echo "Checking out the repository..."
+        git --version
+
+    - name: Configure APT proxy & install Python
+      shell: bash
+      run: |
+        set -eux
+
+        UBUNTU_VERSION=$(grep "VERSION_CODENAME" /etc/os-release | cut -d'=' -f2)
+        ARTIFACTORY_URL="https://artifactory.yc.prod.infra.sravni.market/artifactory/ubuntu_proxy"
+
+        echo "Configuring APT proxy for Artifactory ($UBUNTU_VERSION)..."
+
+        echo "deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} main restricted
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates main restricted
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} universe
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates universe
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} multiverse
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates multiverse
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-backports main restricted universe multiverse
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security main restricted
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security universe
+        deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security multiverse" | sudo tee /etc/apt/sources.list
+
+        echo "Updating APT cache..."
+        sudo apt update -o Acquire::https::Verify-Peer=false -o Acquire::https::Verify-Host=false
+
+        PYTHON_VERSION="${{ inputs.python-version }}"
+
+        echo "Installing Python ${PYTHON_VERSION} from Artifactory..."
+        sudo apt install -y --no-install-recommends \
+            python${PYTHON_VERSION} \
+            ca-certificates
+
+        if [[ ! -f "/usr/bin/python3" ]]; then
+            sudo ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python3
+        fi
+
+        echo "Python ${PYTHON_VERSION} successfully installed from Artifactory!"
+
+    - name: Verify Python installation
+      shell: bash
+      run: |
+        set -eux
+        echo "Using system Python version: ${{ inputs.python-version }}"
+        python3 --version
+
+    - name: Configure pip cache manually
+      shell: bash
+      run: |
+        set -eux
+        export PIP_CACHE_DIR="$HOME/.cache/pip"
+        mkdir -p "$PIP_CACHE_DIR"
+        echo "PIP cache directory: $PIP_CACHE_DIR"
+        echo "PIP_CACHE_DIR=$PIP_CACHE_DIR" >> "$GITHUB_ENV"
+
+    - name: Install dependencies with caching
+      shell: bash
+      run: |
+        set -eux
+
+        echo "Installing BentoML..."
+        if [[ -n "${{ inputs.bentoml-version }}" ]]; then
+          pip install -U "bentoml==${{ inputs.bentoml-version }}" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300
+        else
+          pip install -U "bentoml>=1.0.25" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300
+        fi
+
+        echo "Successfully installed BentoML: $(bentoml --version)"
+
+        echo "Installing hatch and towncrier..."
+        pip install hatch towncrier -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300

.github/workflows/build-and-push-harbor-bentoml.yaml

@@ -63,6 +63,11 @@ jobs:
     name: Bento build
     runs-on: ${{ inputs.runner_bento }}
     needs: [get_projects]
+    env:
+      HF_HUB_ETAG_TIMEOUT: 86400
+      HF_HUB_DOWNLOAD_TIMEOUT: 86400
+      HF_ENDPOINT: https://artifactory.yc.prod.infra.sravni.market/artifactory/api/huggingfaceml/huggingface.co_proxy
+      HF_TOKEN: ${{ secrets.ARTIFACTORY_SVC_TOKEN }} 
 
     strategy:
       max-parallel: 1
@@ -78,19 +83,19 @@ jobs:
           registry: ${{ secrets.HARBOR_REGISTRY }}
           username: ${{ secrets.HARBOR_LOGIN }}
           password: ${{ secrets.HARBOR_PASSWORD }}
-      - uses: bentoml/setup-bentoml-action@v1
+      - uses: sravni/.github/.github/actions/setup@bento-mixed
         with:
           python-version: '3.10'
-          bentoml-version: 'main'
+          bentoml-version: '1.3.14'
       - name: Install requirements
         run: |
           cd ${{ matrix.project.project_path }}
-          pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple
+          pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --timeout 600
       - name: Import model
         run: |
           cd ${{ matrix.project.project_path }}
           python import_model.py
-      - uses: bentoml/build-bento-action@v1
+      - uses: sravni/.github/.github/actions/builder@bento-mixed
         with:
           context: ${{ matrix.project.project_path }}
         id: bento

.github/workflows/build-and-test-harbor.yaml

@@ -135,7 +135,7 @@ jobs:
         if: ${{ env.IMAGE_EXIST == 'false' }}
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: ${{ matrix.project.project_path }}
           file: ${{ matrix.project.dockerfile }}
           build-args: |
             build_number=${{ matrix.project.build_number }}

.github/workflows/get-projects.yaml

@@ -57,15 +57,14 @@ jobs:
 
     outputs:
       projects: >-
-        ${{ steps.get_projects_multi.outputs.projects || 
-            steps.get_projects_mono.outputs.projects || 
-            steps.get_projects_bento.outputs.projects || 
+        ${{ steps.get_projects_multi.outputs.projects ||
+            steps.get_projects_mono.outputs.projects ||
+            steps.get_projects_bento.outputs.projects ||
             steps.get_projects_bento_multi.outputs.projects }}
 
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
-
       - name: Get build number and image version
         run: |
           if [[ -z "${{ inputs.image_version }}" ]]
@@ -87,23 +86,31 @@ jobs:
       - name: Check repo type
         id: repo_type
         run: |
-          if [[ -f './Dockerfile' ]]
-          then
-            echo 'Определили репозиторий как моносервисный'
-            echo "repo_type='mono'" >> $GITHUB_OUTPUT
-          elif [[ -f './bentofile.yaml' ]]
-          then
-            echo 'Определили репозиторий как бенто сервис'
+          # Проверка для bentofile.yaml
+          if [[ -f './bentofile.yaml' ]]; then
+            echo 'Определили репозиторий как бенто-сервисный'
             echo "repo_type='bento'" >> $GITHUB_OUTPUT
-          elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml
-          then
+          elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml; then
             echo 'Определили репозиторий как бенто-мультисервисный'
             echo "repo_type='bento-multi'" >> $GITHUB_OUTPUT
+          fi
+
+          # Если bentofile.yaml найден в inputs.project_path_mask, пропускаем проверку Dockerfile
+          if find "${{ inputs.project_path_mask }}/" -type f -name 'bentofile.yaml' | grep -q '.'; then
+            echo 'Пропускаем проверку Dockerfile, так как найден bentofile.yaml'
+            exit 0
+          fi
+
+          # Проверка для Dockerfile
+          if [[ -f './Dockerfile' ]]; then
+            echo 'Определили репозиторий как моносервисный'
+            echo "repo_type='mono'" >> $GITHUB_OUTPUT
           else
             echo 'Определили репозиторий как мультисервисный'
             echo "repo_type='multi'" >> $GITHUB_OUTPUT
           fi
 
+
       - name: Get projects (multi)
         id: get_projects_multi
         run: |
@@ -192,7 +199,7 @@ jobs:
         run: |
           # Выходим, если репозиторий не бенто мультисервисный
           if [[ ${{ steps.repo_type.outputs.repo_type }} != 'bento-multi' ]]; then exit 0; fi
-      
+
           projects=$(for p in $(find * -maxdepth 3 -path "${{ inputs.project_path_mask }}/bentofile.yaml")
             do
               project_path=$(echo $p | sed -E 's|(.*)/bentofile\.yaml$|\1|')

.github/workflows/publish-to-ml-prod-harbor.yaml

@@ -88,7 +88,9 @@ jobs:
           then
             echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} уже существует'
           else
-            echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создайте его сначала в dev окружении!'
+            echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создадим новый'
+            cd deploy
+            ./helm_create.sh ${{ env.TEAM }} ${{ matrix.project.project_name }} ml_prod
           fi;
 
       - name: Install yq