FIX GOVNO CI/CD

.github/workflows/deploy.yml

@@ -18,9 +18,6 @@ jobs:
     environment: staging
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
       - name: Deploy over SSH
         uses: appleboy/ssh-action@v1.2.0
         with:
@@ -31,26 +28,11 @@ jobs:
           command_timeout: 30m
           script: |
             set -euxo pipefail
-            compose_cmd='docker-compose -p borofone-staging'
             cd /opt/borofone-chat-staging
-            echo "[deploy] $(date -Is) start staging commit=${{ github.sha }}"
             git fetch --prune origin
-            git checkout dev
+            git checkout dev || git checkout -b dev --track origin/dev
             git reset --hard origin/dev
-            systemctl stop borofone-staging || true
-            rm -f deploy/docker/.env
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml up -d --no-recreate postgres redis
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml exec -T postgres sh -lc 'until pg_isready -U "${POSTGRES_USER:-app}" -d postgres -h 127.0.0.1; do sleep 1; done'
-            bash deploy/scripts/backup-compose-data.sh staging borofone-staging deploy/docker/docker-compose.staging.yml borofone_staging borofone-staging_uploads_data
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml exec -T postgres sh -lc 'psql -U "${POSTGRES_USER:-app}" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='\''${POSTGRES_DB:-borofone_staging}'\''" | grep -q 1 || psql -U "${POSTGRES_USER:-app}" -d postgres -c "CREATE DATABASE ${POSTGRES_DB:-borofone_staging}"'
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml run --rm api alembic upgrade head
-            docker ps -q --filter "publish=8001" | xargs -r docker rm -f
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml stop api || true
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml rm -fsv api || true
-            docker ps -aq --filter "label=com.docker.compose.project=borofone-staging" --filter "label=com.docker.compose.service=api" | xargs -r docker rm -f
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml up -d --no-deps --force-recreate --build api
-            $compose_cmd -f deploy/docker/docker-compose.staging.yml ps
-            echo "[deploy] $(date -Is) done staging commit=${{ github.sha }}"
+            SKIP_GIT_SYNC=1 bash deploy/scripts/deploy-stack.sh staging
 
   deploy-production:
     name: Deploy Production
@@ -59,9 +41,6 @@ jobs:
     environment: production
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
       - name: Deploy over SSH
         uses: appleboy/ssh-action@v1.2.0
         with:
@@ -72,24 +51,8 @@ jobs:
           command_timeout: 30m
           script: |
             set -euxo pipefail
-            compose_cmd='docker-compose -p borofone-prod'
             cd /opt/borofone-chat-prod
-            echo "[deploy] $(date -Is) start production commit=${{ github.sha }}"
             git fetch --prune origin
-            git checkout main
+            git checkout main || git checkout -b main --track origin/main
             git reset --hard origin/main
-            systemctl stop borofone-prod || true
-            rm -f deploy/docker/.env
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml up -d --no-recreate postgres redis
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml exec -T postgres sh -lc 'until pg_isready -U "${POSTGRES_USER:-app}" -d postgres -h 127.0.0.1; do sleep 1; done'
-            bash deploy/scripts/backup-compose-data.sh production borofone-prod deploy/docker/docker-compose.prod.yml borofone_prod borofone-prod_uploads_data
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml exec -T postgres sh -lc 'psql -U "${POSTGRES_USER:-app}" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='\''${POSTGRES_DB:-borofone_prod}'\''" | grep -q 1 || psql -U "${POSTGRES_USER:-app}" -d postgres -c "CREATE DATABASE ${POSTGRES_DB:-borofone_prod}"'
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml run --rm api alembic upgrade head
-            docker ps -q --filter "publish=8000" | xargs -r docker rm -f
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml stop api || true
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml rm -fsv api || true
-            docker ps -aq --filter "label=com.docker.compose.project=borofone-prod" --filter "label=com.docker.compose.service=api" | xargs -r docker rm -f
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml up -d --no-deps --force-recreate --build api
-            $compose_cmd -f deploy/docker/docker-compose.prod.yml ps
-            echo "[deploy] $(date -Is) done production commit=${{ github.sha }}"
-
+            SKIP_GIT_SYNC=1 bash deploy/scripts/deploy-stack.sh production

.gitignore

@@ -14,6 +14,9 @@
 /not_tested
 /uploads/attachments
 /docs
+!/docs/
+!/docs/deployment/
+!/docs/deployment/cicd.md
 /data/leaderboard/*.json
 !/data/leaderboard/.gitkeep
 

README.md

@@ -112,8 +112,9 @@ docker compose -f deploy/docker/docker-compose.infra.yml ps
 docker compose -f deploy/docker/docker-compose.infra.yml up -d
 ```
 
-Leaderboard for `tears-of-bfu` is persisted outside the container in `data/leaderboard/leaderboard.json`.
-For staging/production compose stacks this directory is bind-mounted into `/code/pages/web_backend/data`.
+Leaderboard for `tears-of-bfu` is persisted outside the container in `data/leaderboard/leaderboard.json` for local development.
+For staging/production compose stacks, uploads and leaderboard data are stored under `HOST_DATA_ROOT` on the VPS and bind-mounted into the API container.
+The deploy scripts support both `docker compose` and legacy `docker-compose`.
 
 **DOWN infra:**
 
@@ -298,10 +299,10 @@ maybe next time
 
 ## for vps
 
-git pull origin main
-docker-compose -f deploy/docker/docker-compose.prod.yml build --no-cache api
-docker-compose -f deploy/docker/docker-compose.prod.yml down
-docker-compose -f deploy/docker/docker-compose.prod.yml up -d
+```bash
+cd /opt/borofone-chat-prod
+bash deploy/scripts/deploy-stack.sh production
+```
 
 ## Sources
 
@@ -320,7 +321,9 @@ dev  -> staging
 
 - `.github/workflows/deploy.yml` - автодеплой по `push` в `dev` и `main`
 - `deploy/env/.env.production.example` и `deploy/env/.env.staging.example` - шаблоны окружений
-- `deploy/systemd/` - systemd unit-файлы
+- `deploy/scripts/deploy-stack.sh` - единая точка входа для staging/production deploy
+- `deploy/scripts/prepare-persistent-data.sh` - one-time migration of uploads and leaderboard into host storage
+- `deploy/scripts/backup-compose-data.sh` - backup of database, uploads, leaderboard and `.env`
 - `deploy/nginx/borofone.conf` - reverse proxy для production и staging
 - `scripts/setup_vps.sh` - первичная подготовка VPS
 - `docs/deployment/cicd.md` - пошаговые инструкции по setup и security
@@ -336,7 +339,8 @@ merge dev main -> GitHub Actions -> production deploy
 
 1. Добавь GitHub Secrets для `PROD_*` и `STAGING_*`.
 2. Используй `deploy/env/.env.production.example` и `deploy/env/.env.staging.example`, затем скопируй их в `/opt/borofone-chat-prod/.env` и `/opt/borofone-chat-staging/.env`.
-3. Установи `deploy/systemd/*.service` и `deploy/nginx/borofone.conf` на VPS.
-4. Включи branch protection для `main`, запрети прямой push и оставь deploy только через PR.
+3. Заполни `HOST_DATA_ROOT` и `BACKUP_ROOT` абсолютными путями на VPS.
+4. Установи `deploy/nginx/borofone.conf` на VPS.
+5. Включи branch protection для `main`, запрети прямой push и оставь deploy только через PR.
 
 Подробная инструкция: `docs/deployment/cicd.md`.

deploy/docker/Dockerfile

@@ -7,12 +7,11 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY app ./app
 COPY pages ./pages
-COPY uploads ./uploads
 COPY alembic ./alembic
 COPY alembic.ini .
 COPY fix_sequences.sql .
 
-RUN mkdir -p /code/pages/web_backend/data /code/uploads
+RUN mkdir -p /code/pages/web_backend/data /code/uploads /code/uploads/avatars /code/uploads/attachments
 
 ENV APP_PORT=8000
 

deploy/docker/docker-compose.prod.yml

@@ -36,8 +36,16 @@ services:
     ports:
       - "127.0.0.1:8000:8000"
     volumes:
-      - uploads_data:/code/uploads
-      - ../../data/leaderboard:/code/pages/web_backend/data
+      - ${HOST_DATA_ROOT:?HOST_DATA_ROOT is required}/uploads:/code/uploads
+      - ${HOST_DATA_ROOT:?HOST_DATA_ROOT is required}/leaderboard:/code/pages/web_backend/data
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - 'python -c "import os, urllib.request; urllib.request.urlopen(''http://127.0.0.1:{}/app-config.js''.format(os.environ.get(''APP_PORT'', ''8000'')), timeout=5)"'
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 10s
     depends_on:
       postgres:
         condition: service_healthy
@@ -51,8 +59,6 @@ volumes:
     name: borofone-prod_postgres_data
   redis_data:
     name: borofone-prod_redis_data
-  uploads_data:
-    name: borofone-prod_uploads_data
 
 networks:
   prod_network:

deploy/docker/docker-compose.staging.yml

@@ -36,8 +36,16 @@ services:
     ports:
       - "127.0.0.1:8001:8001"
     volumes:
-      - uploads_data:/code/uploads
-      - ../../data/leaderboard:/code/pages/web_backend/data
+      - ${HOST_DATA_ROOT:?HOST_DATA_ROOT is required}/uploads:/code/uploads
+      - ${HOST_DATA_ROOT:?HOST_DATA_ROOT is required}/leaderboard:/code/pages/web_backend/data
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - 'python -c "import os, urllib.request; urllib.request.urlopen(''http://127.0.0.1:{}/app-config.js''.format(os.environ.get(''APP_PORT'', ''8000'')), timeout=5)"'
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 10s
     depends_on:
       postgres:
         condition: service_healthy
@@ -51,8 +59,6 @@ volumes:
     name: borofone-staging_postgres_data
   redis_data:
     name: borofone-staging_redis_data
-  uploads_data:
-    name: borofone-staging_uploads_data
 
 networks:
   staging_network:

deploy/env/.env.production.example

@@ -21,13 +21,17 @@ COOKIE_SAMESITE=lax
 ACCESS_TOKEN_EXPIRE_DAYS=30
 REFRESH_TOKEN_EXPIRE_DAYS=30
 
+HOST_DATA_ROOT=/var/lib/borofone/production
+BACKUP_ROOT=/var/backups/borofone
+
+# Always backed by host bind mount from ${HOST_DATA_ROOT}/uploads.
 UPLOADS_DIR=/code/uploads
 PAGES_DIR=pages
 FAVICON_PATH=favicon.ico
 MAIN_PAGE_PATH=main.html
 LOGIN_PAGE_PATH=login.html
 REGISTER_PAGE_PATH=register.html
-# Leaderboard JSON is persisted via docker bind mount: ../../data/leaderboard -> /code/pages/web_backend/data
+# Leaderboard files are always backed by host bind mount from ${HOST_DATA_ROOT}/leaderboard.
 
 MAX_AVATAR_BYTES=3145728
 MAX_UPLOAD_FILE_SIZE=10485760

deploy/env/.env.staging.example

@@ -21,13 +21,17 @@ COOKIE_SAMESITE=lax
 ACCESS_TOKEN_EXPIRE_DAYS=30
 REFRESH_TOKEN_EXPIRE_DAYS=30
 
+HOST_DATA_ROOT=/var/lib/borofone/staging
+BACKUP_ROOT=/var/backups/borofone
+
+# Always backed by host bind mount from ${HOST_DATA_ROOT}/uploads.
 UPLOADS_DIR=/code/uploads
 PAGES_DIR=pages
 FAVICON_PATH=favicon.ico
 MAIN_PAGE_PATH=main.html
 LOGIN_PAGE_PATH=login.html
 REGISTER_PAGE_PATH=register.html
-# Leaderboard JSON is persisted via docker bind mount: ../../data/leaderboard -> /code/pages/web_backend/data
+# Leaderboard files are always backed by host bind mount from ${HOST_DATA_ROOT}/leaderboard.
 
 MAX_AVATAR_BYTES=3145728
 MAX_UPLOAD_FILE_SIZE=10485760

deploy/scripts/backup-compose-data.sh

@@ -12,12 +12,61 @@ compose_file="$3"
 db_name="$4"
 uploads_volume="$5"
 
-backup_root="backups/${env_name}"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(cd "${script_dir}/../.." && pwd)"
+
+resolve_compose_cmd() {
+  if docker compose version >/dev/null 2>&1; then
+    COMPOSE_BIN=(docker compose)
+    return 0
+  fi
+
+  if command -v docker-compose >/dev/null 2>&1; then
+    COMPOSE_BIN=(docker-compose)
+    return 0
+  fi
+
+  echo "docker compose or docker-compose is required" >&2
+  exit 1
+}
+
+if [ ! -f "${repo_root}/.env" ]; then
+  echo "missing ${repo_root}/.env" >&2
+  exit 1
+fi
+
+set -a
+. "${repo_root}/.env"
+set +a
+
+: "${BACKUP_ROOT:?BACKUP_ROOT must be set in .env}"
+: "${HOST_DATA_ROOT:?HOST_DATA_ROOT must be set in .env}"
+
+backup_root="${BACKUP_ROOT%/}/${env_name}"
 timestamp="$(date +%Y%m%d-%H%M%S)"
 target_dir="${backup_root}/${timestamp}"
 mkdir -p "${target_dir}"
 
-compose_cmd=(docker-compose -p "${compose_project}" -f "${compose_file}")
+resolve_compose_cmd
+compose_cmd=("${COMPOSE_BIN[@]}" -p "${compose_project}" -f "${compose_file}")
+host_uploads_dir="${HOST_DATA_ROOT%/}/uploads"
+host_leaderboard_dir="${HOST_DATA_ROOT%/}/leaderboard"
+legacy_leaderboard_dir="${repo_root}/data/leaderboard"
+
+tar_directory() {
+  local source_dir="$1"
+  local target_archive="$2"
+
+  if [ ! -d "${source_dir}" ]; then
+    return 1
+  fi
+
+  if [ -z "$(find "${source_dir}" -mindepth 1 -print -quit 2>/dev/null)" ]; then
+    return 1
+  fi
+
+  tar -C "${source_dir}" -czf "${target_archive}" .
+}
 
 "${compose_cmd[@]}" up -d --no-recreate postgres redis
 "${compose_cmd[@]}" exec -T postgres sh -lc 'until pg_isready -U "${POSTGRES_USER:-app}" -d postgres -h 127.0.0.1; do sleep 1; done'
@@ -26,13 +75,21 @@ if "${compose_cmd[@]}" exec -T postgres sh -lc "psql -U \"\${POSTGRES_USER:-app}
   "${compose_cmd[@]}" exec -T postgres sh -lc "pg_dump -U \"\${POSTGRES_USER:-app}\" -d \"${db_name}\" --clean --if-exists --no-owner --no-privileges" | gzip -9 > "${target_dir}/${db_name}.sql.gz"
 fi
 
-if docker volume inspect "${uploads_volume}" >/dev/null 2>&1; then
+if tar_directory "${host_uploads_dir}" "${target_dir}/uploads.tar.gz"; then
+  :
+elif docker volume inspect "${uploads_volume}" >/dev/null 2>&1; then
   docker run --rm \
     -v "${uploads_volume}:/source:ro" \
-    -v "${PWD}/${target_dir}:/backup" \
+    -v "${target_dir}:/backup" \
     alpine:3.20 \
     sh -lc 'cd /source && tar -czf /backup/uploads.tar.gz .'
 fi
 
+if ! tar_directory "${host_leaderboard_dir}" "${target_dir}/leaderboard.tar.gz"; then
+  tar_directory "${legacy_leaderboard_dir}" "${target_dir}/leaderboard.tar.gz" || true
+fi
+
+cp "${repo_root}/.env" "${target_dir}/.env"
+
 find "${backup_root}" -mindepth 1 -maxdepth 1 -type d | sort | head -n -10 | xargs -r rm -rf
 echo "[backup] ${env_name} saved to ${target_dir}"