diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index de659e864..bac680e7b 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -7,23 +7,82 @@ on: workflow_dispatch: jobs: - check-version: + create-nightly-tag: if: github.repository == 'langflow-ai/openrag' runs-on: ubuntu-latest + permissions: + contents: write outputs: - nightly_version: ${{ steps.vars.outputs.nightly_version }} + nightly_version: ${{ steps.ensure_unique_tag.outputs.nightly_tag }} steps: - - name: Determine Nightly Version - id: vars + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies for tagging script + run: uv pip install requests packaging + + - name: Generate nightly tag + id: generate_tag + run: | + TAG=$(python3 scripts/ci/pypi_nightly_tag.py) + echo "nightly_tag=$TAG" >> $GITHUB_OUTPUT + echo "nightly_tag=$TAG" + + - name: Ensure unique nightly tag + id: ensure_unique_tag + run: | + BASE_TAG="${{ steps.generate_tag.outputs.nightly_tag }}" + TAG="$BASE_TAG" + # If the tag already exists on the remote, resolve collision using .devN suffixes (PEP 440 compliant) + if git ls-remote --exit-code --tags origin "$TAG" >/dev/null 2>&1; then + echo "Base tag '$TAG' already exists on remote. Searching for available .devN suffix..." + MAX_DEV=20 + FOUND=0 + i=1 + while [ "$i" -le "$MAX_DEV" ]; do + CANDIDATE="${BASE_TAG}.dev${i}" + if ! git ls-remote --exit-code --tags origin "$CANDIDATE" >/dev/null 2>&1; then + TAG="$CANDIDATE" + FOUND=1 + break + fi + i=$((i + 1)) + done + if [ "$FOUND" -ne 1 ]; then + echo "Error: Unable to find an available .devN tag for base tag '$BASE_TAG' after checking ${MAX_DEV} candidates." >&2 + exit 1 + fi + fi + echo "Resolved nightly tag: $TAG" + echo "nightly_tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Update pyproject.toml run: | - DATE_PART=$(date -u +%y%m%d) - # Use the GitHub run number to create a unique, auto-incrementing suffix - NIGHTLY_VERSION="nightly-${DATE_PART}.${{ github.run_number }}.${{ github.run_attempt }}" - echo "nightly_version=$NIGHTLY_VERSION" >> "$GITHUB_OUTPUT" - echo "Nightly version: $NIGHTLY_VERSION" + python3 scripts/ci/update_pyproject_combined.py main ${{ steps.ensure_unique_tag.outputs.nightly_tag }} + + - name: Commit and push tag + run: | + git config --global user.email "bot-nightly-builds@openrag.org" + git config --global user.name "OpenRAG Bot" + + git add pyproject.toml + git commit -m "Update version and project name for nightly ${{ steps.ensure_unique_tag.outputs.nightly_tag }}" + + git tag ${{ steps.ensure_unique_tag.outputs.nightly_tag }} + git push origin ${{ steps.ensure_unique_tag.outputs.nightly_tag }} build: - needs: check-version + needs: create-nightly-tag strategy: fail-fast: false matrix: @@ -73,8 +132,11 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - name: Checkout + - name: Checkout nightly tag uses: actions/checkout@v4 + with: + ref: ${{ needs.create-nightly-tag.outputs.nightly_version }} + fetch-depth: 0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -92,7 +154,7 @@ jobs: file: ${{ matrix.file }} platforms: ${{ matrix.platform }} push: true - tags: langflowai/${{ matrix.service }}:${{ needs.check-version.outputs.nightly_version }}-${{ matrix.arch }} + tags: langflowai/${{ matrix.service }}:${{ needs.create-nightly-tag.outputs.nightly_version }}-${{ matrix.arch }} cache-from: type=gha,scope=nightly-${{ matrix.image }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=nightly-${{ matrix.image }}-${{ matrix.arch }} @@ -107,7 +169,7 @@ jobs: path: service-${{ matrix.service }}-${{ matrix.arch }}.txt manifest: - needs: [build, check-version] + needs: [build, create-nightly-tag] runs-on: ubuntu-latest steps: - name: Download built service artifacts @@ -125,7 +187,7 @@ jobs: - name: Create and push multi-arch manifests run: | NS="langflowai" - VERSION=${{ needs.check-version.outputs.nightly_version }} + VERSION=${{ needs.create-nightly-tag.outputs.nightly_version }} # Derive the list of services from the artifacts produced by the build matrix mapfile -t SERVICES < <(find . -name 'service-*.txt' -print0 | xargs -0 cat | sort -u) @@ -144,7 +206,7 @@ jobs: $NS/$SVC:$VERSION-arm64 done publish-pypi: - needs: [check-version] + needs: [create-nightly-tag] runs-on: ubuntu-latest if: github.repository == 'langflow-ai/openrag' steps: @@ -161,15 +223,14 @@ jobs: - name: Update version for nightly run: | - python3 -c " - import re - with open('pyproject.toml', 'r') as f: - content = f.read() - # Append .dev + run_number + run_attempt to the version (PEP 440 compliant and unique per attempt) - new_content = re.sub(r'^version = \"([^\"]+)\"', r'version = \"\1.dev${{ github.run_number }}${{ github.run_attempt }}\"', content, flags=re.M) - with open('pyproject.toml', 'w') as f: - f.write(new_content) - " + # The version has already been updated in create-nightly-tag job + # but we need to ensure we are on the correct tag/commit if needed. + # However, publish-pypi runs on the same branch by default. + # In the create-nightly-tag job we pushed a tag, here we should probably checkout that tag + # or just use the updated pyproject.toml if it's shared (it's not). + # So we need to checkout the tag. + git fetch origin "refs/tags/${{ needs.create-nightly-tag.outputs.nightly_version }}:refs/tags/${{ needs.create-nightly-tag.outputs.nightly_version }}" + git checkout ${{ needs.create-nightly-tag.outputs.nightly_version }} - name: Build wheel and source distribution run: | diff --git a/scripts/ci/pypi_nightly_tag.py b/scripts/ci/pypi_nightly_tag.py new file mode 100644 index 000000000..63cb5b6ad --- /dev/null +++ b/scripts/ci/pypi_nightly_tag.py @@ -0,0 +1,62 @@ +import sys +import requests +from packaging.version import Version +from pathlib import Path +import tomllib +from typing import Optional + +PYPI_OPENRAG_NIGHTLY_URL = "https://pypi.org/pypi/openrag-nightly/json" +PYPI_OPENRAG_URL = "https://pypi.org/pypi/openrag/json" + +def get_latest_published_version(is_nightly: bool) -> Optional[Version]: + url = PYPI_OPENRAG_NIGHTLY_URL if is_nightly else PYPI_OPENRAG_URL + res = requests.get(url, timeout=10) + if res.status_code == 404: + return None + res.raise_for_status() + try: + version_str = res.json()["info"]["version"] + except Exception as e: + msg = "Got unexpected response from PyPI" + raise RuntimeError(msg) from e + return Version(version_str) + +def create_tag(): + # Read version from pyproject.toml + pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml" + with open(pyproject_path, "rb") as f: + pyproject_data = tomllib.load(f) + + current_version_str = pyproject_data["project"]["version"] + current_version = Version(current_version_str) + + try: + current_nightly_version = get_latest_published_version(is_nightly=True) + except (requests.RequestException, KeyError, ValueError): + current_nightly_version = None + + build_number = "0" + latest_base_version = current_version.base_version + nightly_base_version = current_nightly_version.base_version if current_nightly_version else None + + if latest_base_version == nightly_base_version: + dev_number = (current_nightly_version.dev if current_nightly_version.dev is not None else -1) if current_nightly_version else -1 + build_number = str(dev_number + 1) + + # Build PEP 440-compliant nightly version (without leading "v") + nightly_version_str = f"{latest_base_version}.dev{build_number}" + + # Verify PEP440 + Version(nightly_version_str) + + # Git tag uses a leading "v" prefix + new_nightly_version = f"v{nightly_version_str}" + return new_nightly_version + +if __name__ == "__main__": + try: + tag = create_tag() + print(tag) + except Exception as e: + print(f"Error creating tag: {e}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/ci/update_pyproject_combined.py b/scripts/ci/update_pyproject_combined.py new file mode 100644 index 000000000..01949123f --- /dev/null +++ b/scripts/ci/update_pyproject_combined.py @@ -0,0 +1,30 @@ +import sys +from pathlib import Path + +# Add current dir to sys.path +current_dir = Path(__file__).resolve().parent +current_dir_str = str(current_dir) +if current_dir_str not in sys.path: + sys.path.append(current_dir_str) + +from update_pyproject_name import update_pyproject_name +from update_pyproject_version import update_version + +def main(): + if len(sys.argv) != 3: + print("Usage: update_pyproject_combined.py main ") + sys.exit(1) + + mode = sys.argv[1] + main_tag = sys.argv[2] + + if mode != "main": + print("Only 'main' mode is supported") + sys.exit(1) + + # Update name and version for openrag + update_pyproject_name("openrag-nightly") + update_version(main_tag) + +if __name__ == "__main__": + main() diff --git a/scripts/ci/update_pyproject_name.py b/scripts/ci/update_pyproject_name.py new file mode 100644 index 000000000..20d46a261 --- /dev/null +++ b/scripts/ci/update_pyproject_name.py @@ -0,0 +1,29 @@ +import re +import sys +from pathlib import Path + +def update_pyproject_name(new_name: str): + path = Path("pyproject.toml") + if not path.exists(): + print("File pyproject.toml not found") + raise SystemExit(1) + + content = path.read_text() + new_content = re.sub(r'^name = "[^"]+"', f'name = "{new_name}"', content, flags=re.M) + + # Fail if the name pattern was not found / no substitution was made + if new_content == content: + print( + 'Error: Could not find a line matching `name = "..."` in pyproject.toml to update.', + file=sys.stderr, + ) + sys.exit(1) + + path.write_text(new_content) + print(f"Updated name in pyproject.toml to {new_name}") + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: update_pyproject_name.py ") + sys.exit(1) + update_pyproject_name(sys.argv[1]) diff --git a/scripts/ci/update_pyproject_version.py b/scripts/ci/update_pyproject_version.py new file mode 100644 index 000000000..d4bd29b91 --- /dev/null +++ b/scripts/ci/update_pyproject_version.py @@ -0,0 +1,48 @@ +import re +import sys +from pathlib import Path +from packaging.version import Version, InvalidVersion + +def update_version(new_version): + pyproject_path = Path("pyproject.toml") + with open(pyproject_path, "r") as f: + content = f.read() + + # Update the version field + # Removes 'v' prefix if present from tag + clean_version = new_version.lstrip('v') + + # Validate that the resulting version is a valid PEP 440 version + try: + Version(clean_version) + except InvalidVersion: + print( + f"Error: '{clean_version}' is not a valid PEP 440 version after stripping any leading 'v'.", + file=sys.stderr, + ) + sys.exit(1) + + new_content = re.sub( + r'^version = "[^"]+"', + f'version = "{clean_version}"', + content, + flags=re.M, + ) + + # Fail if the version pattern was not found / no substitution was made + if new_content == content: + print( + 'Error: Could not find a line matching `version = "..."` in pyproject.toml to update.', + file=sys.stderr, + ) + sys.exit(1) + + with open(pyproject_path, "w") as f: + f.write(new_content) + print(f"Updated pyproject.toml version to {clean_version}") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python update_pyproject_version.py ") + sys.exit(1) + update_version(sys.argv[1]) diff --git a/src/tui/utils/version_check.py b/src/tui/utils/version_check.py index 16c5d884f..334832ac7 100644 --- a/src/tui/utils/version_check.py +++ b/src/tui/utils/version_check.py @@ -88,15 +88,18 @@ def get_current_version() -> str: Returns: Version string or "unknown" if not available """ - try: - from importlib.metadata import version - return version("openrag") - except Exception: + for dist_name in ["openrag", "openrag-nightly"]: try: - from tui import __version__ - return __version__ + from importlib.metadata import version + return version(dist_name) except Exception: - return "unknown" + continue + + try: + from tui import __version__ + return __version__ + except Exception: + return "unknown" def compare_versions(version1: str, version2: str) -> int: diff --git a/src/utils/version_utils.py b/src/utils/version_utils.py index 061db5b73..09c22a13a 100644 --- a/src/utils/version_utils.py +++ b/src/utils/version_utils.py @@ -8,7 +8,12 @@ def _get_openrag_version() -> str: from importlib.metadata import version, PackageNotFoundError try: - return version("openrag") + for dist_name in ["openrag", "openrag-nightly"]: + try: + return version(dist_name) + except PackageNotFoundError: + continue + raise PackageNotFoundError("openrag") except PackageNotFoundError: # Fallback: try to read from pyproject.toml if package not installed (dev mode) try: