diff --git a/.github/actions/builder/action.yml b/.github/actions/builder/action.yml new file mode 100644 index 0000000..22cc4fc --- /dev/null +++ b/.github/actions/builder/action.yml @@ -0,0 +1,53 @@ +# https://help.github.com/en/articles/metadata-syntax-for-github-actions +name: Build Bento bundle with GitHub Action +description: Build Bento with GitHub Action +author: bentoml +branding: + icon: 'anchor' + color: 'blue' +inputs: + bentofile: + description: "Optional path to bentofile.yaml" + required: false + version: + description: "Optional version for this given Bento. By default this will be auto-generated" + required: false + context: + description: "Build's context is the set of files located in the specified PATH or URL" + required: false + default: '.' +outputs: + bento-tag: + description: 'Bento tag' + value: ${{ steps.build.outputs.bento-tag }} + bento-name: + description: 'Bento name' + value: ${{ steps.build.outputs.bento-name }} + bento-version: + description: 'Bento version' + value: ${{ steps.build.outputs.bento-version }} + metadata: + description: 'Build result metadata (equivalent of `bentoml get :`)' + value: ${{ steps.build.outputs.bento-metadata }} +runs: + using: 'composite' + steps: + - shell: bash + id: build + run: | + build_args=() + if [[ -n "${{ inputs.bentofile }}" ]]; then + build_args+=(--bentofile ${{ inputs.bentofile }}) + fi + if [[ -n "${{ inputs.version }}" ]]; then + build_args+=(--version ${{ inputs.version }}) + fi + bento_tag=$(bentoml build -o tag ${build_args[@]} ${{ inputs.context }}) + bento_tag=${bento_tag##__tag__:} + echo "bento-tag=$bento_tag" >> $GITHUB_OUTPUT + echo "bento-name=$(echo $bento_tag | cut -d: -f1)" >> $GITHUB_OUTPUT + echo "bento-version=$(echo $bento_tag | cut -d: -f2)" >> $GITHUB_OUTPUT + # Multiline string for gh action https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + echo "bento-metadata<> $GITHUB_OUTPUT + echo "$(bentoml get $bento_tag -o json)" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..15c3d57 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,97 @@ +name: "Setup BentoML" +description: "Bootstrap and setup base environment for BentoML" +branding: + icon: 'box' + color: 'orange' +inputs: + bentoml-version: + description: The version of BentoML to use. Leave it empty to use the latest version, set to 'main' to use 'nightly' + required: false + default: "" + python-version: + description: Default Python version + required: false + default: "3.10" + architecture: + description: 'Which architecture to run on' + required: false + default: x64 +runs: + using: composite + steps: + - name: Checkout source + shell: bash + run: | + set -eux + echo "Checking out the repository..." + git --version + + - name: Configure APT proxy & install Python + shell: bash + run: | + set -eux + + UBUNTU_VERSION=$(grep "VERSION_CODENAME" /etc/os-release | cut -d'=' -f2) + ARTIFACTORY_URL="https://artifactory.yc.prod.infra.sravni.market/artifactory/ubuntu_proxy" + + echo "Configuring APT proxy for Artifactory ($UBUNTU_VERSION)..." + + echo "deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION} multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-updates multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-backports main restricted universe multiverse + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security main restricted + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security universe + deb ${ARTIFACTORY_URL} ${UBUNTU_VERSION}-security multiverse" | sudo tee /etc/apt/sources.list + + echo "Updating APT cache..." + sudo apt update -o Acquire::https::Verify-Peer=false -o Acquire::https::Verify-Host=false + + PYTHON_VERSION="${{ inputs.python-version }}" + + echo "Installing Python ${PYTHON_VERSION} from Artifactory..." + sudo apt install -y --no-install-recommends \ + python${PYTHON_VERSION} \ + ca-certificates + + if [[ ! -f "/usr/bin/python3" ]]; then + sudo ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 + fi + + echo "Python ${PYTHON_VERSION} successfully installed from Artifactory!" + + - name: Verify Python installation + shell: bash + run: | + set -eux + echo "Using system Python version: ${{ inputs.python-version }}" + python3 --version + + - name: Configure pip cache manually + shell: bash + run: | + set -eux + export PIP_CACHE_DIR="$HOME/.cache/pip" + mkdir -p "$PIP_CACHE_DIR" + echo "PIP cache directory: $PIP_CACHE_DIR" + echo "PIP_CACHE_DIR=$PIP_CACHE_DIR" >> "$GITHUB_ENV" + + - name: Install dependencies with caching + shell: bash + run: | + set -eux + + echo "Installing BentoML..." + if [[ -n "${{ inputs.bentoml-version }}" ]]; then + pip install -U "bentoml==${{ inputs.bentoml-version }}" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300 + else + pip install -U "bentoml>=1.0.25" -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300 + fi + + echo "Successfully installed BentoML: $(bentoml --version)" + + echo "Installing hatch and towncrier..." + pip install hatch towncrier -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --cache-dir="$PIP_CACHE_DIR" --timeout=300 diff --git a/.github/workflows/build-and-push-harbor-bentoml.yaml b/.github/workflows/build-and-push-harbor-bentoml.yaml index 843e34f..ed6eac6 100644 --- a/.github/workflows/build-and-push-harbor-bentoml.yaml +++ b/.github/workflows/build-and-push-harbor-bentoml.yaml @@ -63,6 +63,11 @@ jobs: name: Bento build runs-on: ${{ inputs.runner_bento }} needs: [get_projects] + env: + HF_HUB_ETAG_TIMEOUT: 86400 + HF_HUB_DOWNLOAD_TIMEOUT: 86400 + HF_ENDPOINT: https://artifactory.yc.prod.infra.sravni.market/artifactory/api/huggingfaceml/huggingface.co_proxy + HF_TOKEN: ${{ secrets.ARTIFACTORY_SVC_TOKEN }} strategy: max-parallel: 1 @@ -78,19 +83,19 @@ jobs: registry: ${{ secrets.HARBOR_REGISTRY }} username: ${{ secrets.HARBOR_LOGIN }} password: ${{ secrets.HARBOR_PASSWORD }} - - uses: bentoml/setup-bentoml-action@v1 + - uses: sravni/.github/.github/actions/setup@bento-mixed with: python-version: '3.10' - bentoml-version: 'main' + bentoml-version: '1.3.14' - name: Install requirements run: | cd ${{ matrix.project.project_path }} - pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple + pip install -r requirements.txt -i https://artifactory.yc.prod.infra.sravni.market/artifactory/api/pypi/pypi.org_proxy/simple --timeout 600 - name: Import model run: | cd ${{ matrix.project.project_path }} python import_model.py - - uses: bentoml/build-bento-action@v1 + - uses: sravni/.github/.github/actions/builder@bento-mixed with: context: ${{ matrix.project.project_path }} id: bento diff --git a/.github/workflows/build-and-test-harbor.yaml b/.github/workflows/build-and-test-harbor.yaml index 9268d8a..9118423 100644 --- a/.github/workflows/build-and-test-harbor.yaml +++ b/.github/workflows/build-and-test-harbor.yaml @@ -135,7 +135,7 @@ jobs: if: ${{ env.IMAGE_EXIST == 'false' }} uses: docker/build-push-action@v6 with: - context: . + context: ${{ matrix.project.project_path }} file: ${{ matrix.project.dockerfile }} build-args: | build_number=${{ matrix.project.build_number }} diff --git a/.github/workflows/get-projects.yaml b/.github/workflows/get-projects.yaml index 4aa1941..f5aff2b 100644 --- a/.github/workflows/get-projects.yaml +++ b/.github/workflows/get-projects.yaml @@ -57,15 +57,14 @@ jobs: outputs: projects: >- - ${{ steps.get_projects_multi.outputs.projects || - steps.get_projects_mono.outputs.projects || - steps.get_projects_bento.outputs.projects || + ${{ steps.get_projects_multi.outputs.projects || + steps.get_projects_mono.outputs.projects || + steps.get_projects_bento.outputs.projects || steps.get_projects_bento_multi.outputs.projects }} steps: - name: Checkout source uses: actions/checkout@v4 - - name: Get build number and image version run: | if [[ -z "${{ inputs.image_version }}" ]] @@ -87,23 +86,31 @@ jobs: - name: Check repo type id: repo_type run: | - if [[ -f './Dockerfile' ]] - then - echo 'Определили репозиторий как моносервисный' - echo "repo_type='mono'" >> $GITHUB_OUTPUT - elif [[ -f './bentofile.yaml' ]] - then - echo 'Определили репозиторий как бенто сервис' + # Проверка для bentofile.yaml + if [[ -f './bentofile.yaml' ]]; then + echo 'Определили репозиторий как бенто-сервисный' echo "repo_type='bento'" >> $GITHUB_OUTPUT - elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml - then + elif find . -type f -name 'bentofile.yaml' | grep -q -m 1 bentofile.yaml; then echo 'Определили репозиторий как бенто-мультисервисный' echo "repo_type='bento-multi'" >> $GITHUB_OUTPUT + fi + + # Если bentofile.yaml найден в inputs.project_path_mask, пропускаем проверку Dockerfile + if find "${{ inputs.project_path_mask }}/" -type f -name 'bentofile.yaml' | grep -q '.'; then + echo 'Пропускаем проверку Dockerfile, так как найден bentofile.yaml' + exit 0 + fi + + # Проверка для Dockerfile + if [[ -f './Dockerfile' ]]; then + echo 'Определили репозиторий как моносервисный' + echo "repo_type='mono'" >> $GITHUB_OUTPUT else echo 'Определили репозиторий как мультисервисный' echo "repo_type='multi'" >> $GITHUB_OUTPUT fi + - name: Get projects (multi) id: get_projects_multi run: | @@ -192,7 +199,7 @@ jobs: run: | # Выходим, если репозиторий не бенто мультисервисный if [[ ${{ steps.repo_type.outputs.repo_type }} != 'bento-multi' ]]; then exit 0; fi - + projects=$(for p in $(find * -maxdepth 3 -path "${{ inputs.project_path_mask }}/bentofile.yaml") do project_path=$(echo $p | sed -E 's|(.*)/bentofile\.yaml$|\1|') diff --git a/.github/workflows/publish-to-ml-prod-harbor.yaml b/.github/workflows/publish-to-ml-prod-harbor.yaml index dfa5350..5e366ae 100644 --- a/.github/workflows/publish-to-ml-prod-harbor.yaml +++ b/.github/workflows/publish-to-ml-prod-harbor.yaml @@ -88,7 +88,9 @@ jobs: then echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} уже существует' else - echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создайте его сначала в dev окружении!' + echo 'Проект ${{ env.REPO_PATH }}/${{ matrix.project.project_name }} не существует, создадим новый' + cd deploy + ./helm_create.sh ${{ env.TEAM }} ${{ matrix.project.project_name }} ml_prod fi; - name: Install yq