fix: updated logic to update versions with scripts for nightly

.github/workflows/nightly-build.yml

@@ -7,23 +7,82 @@ on:
   workflow_dispatch:
 
 jobs:
-  check-version:
+  create-nightly-tag:
     if: github.repository == 'langflow-ai/openrag'
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     outputs:
-      nightly_version: ${{ steps.vars.outputs.nightly_version }}
+      nightly_version: ${{ steps.ensure_unique_tag.outputs.nightly_tag }}
     steps:
-      - name: Determine Nightly Version
-        id: vars
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install dependencies for tagging script
+        run: uv pip install requests packaging
+
+      - name: Generate nightly tag
+        id: generate_tag
+        run: |
+          TAG=$(python3 scripts/ci/pypi_nightly_tag.py)
+          echo "nightly_tag=$TAG" >> $GITHUB_OUTPUT
+          echo "nightly_tag=$TAG"
+
+      - name: Ensure unique nightly tag
+        id: ensure_unique_tag
+        run: |
+          BASE_TAG="${{ steps.generate_tag.outputs.nightly_tag }}"
+          TAG="$BASE_TAG"
+          # If the tag already exists on the remote, resolve collision using .devN suffixes (PEP 440 compliant)
+          if git ls-remote --exit-code --tags origin "$TAG" >/dev/null 2>&1; then
+            echo "Base tag '$TAG' already exists on remote. Searching for available .devN suffix..."
+            MAX_DEV=20
+            FOUND=0
+            i=1
+            while [ "$i" -le "$MAX_DEV" ]; do
+              CANDIDATE="${BASE_TAG}.dev${i}"
+              if ! git ls-remote --exit-code --tags origin "$CANDIDATE" >/dev/null 2>&1; then
+                TAG="$CANDIDATE"
+                FOUND=1
+                break
+              fi
+              i=$((i + 1))
+            done
+            if [ "$FOUND" -ne 1 ]; then
+              echo "Error: Unable to find an available .devN tag for base tag '$BASE_TAG' after checking ${MAX_DEV} candidates." >&2
+              exit 1
+            fi
+          fi
+          echo "Resolved nightly tag: $TAG"
+          echo "nightly_tag=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Update pyproject.toml
         run: |
-          DATE_PART=$(date -u +%y%m%d)
-          # Use the GitHub run number to create a unique, auto-incrementing suffix
-          NIGHTLY_VERSION="nightly-${DATE_PART}.${{ github.run_number }}.${{ github.run_attempt }}"
-          echo "nightly_version=$NIGHTLY_VERSION" >> "$GITHUB_OUTPUT"
-          echo "Nightly version: $NIGHTLY_VERSION"
+          python3 scripts/ci/update_pyproject_combined.py main ${{ steps.ensure_unique_tag.outputs.nightly_tag }}
+
+      - name: Commit and push tag
+        run: |
+          git config --global user.email "bot-nightly-builds@openrag.org"
+          git config --global user.name "OpenRAG Bot"
+
+          git add pyproject.toml
+          git commit -m "Update version and project name for nightly ${{ steps.ensure_unique_tag.outputs.nightly_tag }}"
+
+          git tag ${{ steps.ensure_unique_tag.outputs.nightly_tag }}
+          git push origin ${{ steps.ensure_unique_tag.outputs.nightly_tag }}
 
   build:
-    needs: check-version
+    needs: create-nightly-tag
     strategy:
       fail-fast: false
       matrix:
@@ -73,8 +132,11 @@ jobs:
 
     runs-on: ${{ matrix.runs-on }}
     steps:
-      - name: Checkout
+      - name: Checkout nightly tag
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.create-nightly-tag.outputs.nightly_version }}
+          fetch-depth: 0
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -92,7 +154,7 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ matrix.platform }}
           push: true
-          tags: langflowai/${{ matrix.service }}:${{ needs.check-version.outputs.nightly_version }}-${{ matrix.arch }}
+          tags: langflowai/${{ matrix.service }}:${{ needs.create-nightly-tag.outputs.nightly_version }}-${{ matrix.arch }}
           cache-from: type=gha,scope=nightly-${{ matrix.image }}-${{ matrix.arch }}
           cache-to: type=gha,mode=max,scope=nightly-${{ matrix.image }}-${{ matrix.arch }}
 
@@ -107,7 +169,7 @@ jobs:
           path: service-${{ matrix.service }}-${{ matrix.arch }}.txt
 
   manifest:
-    needs: [build, check-version]
+    needs: [build, create-nightly-tag]
     runs-on: ubuntu-latest
     steps:
       - name: Download built service artifacts
@@ -125,7 +187,7 @@ jobs:
       - name: Create and push multi-arch manifests
         run: |
           NS="langflowai"
-          VERSION=${{ needs.check-version.outputs.nightly_version }}
+          VERSION=${{ needs.create-nightly-tag.outputs.nightly_version }}
 
           # Derive the list of services from the artifacts produced by the build matrix
           mapfile -t SERVICES < <(find . -name 'service-*.txt' -print0 | xargs -0 cat | sort -u)
@@ -144,7 +206,7 @@ jobs:
               $NS/$SVC:$VERSION-arm64
           done
   publish-pypi:
-    needs: [check-version]
+    needs: [create-nightly-tag]
     runs-on: ubuntu-latest
     if: github.repository == 'langflow-ai/openrag'
     steps:
@@ -161,15 +223,14 @@ jobs:
 
       - name: Update version for nightly
         run: |
-          python3 -c "
-          import re
-          with open('pyproject.toml', 'r') as f:
-              content = f.read()
-          # Append .dev + run_number + run_attempt to the version (PEP 440 compliant and unique per attempt)
-          new_content = re.sub(r'^version = \"([^\"]+)\"', r'version = \"\1.dev${{ github.run_number }}${{ github.run_attempt }}\"', content, flags=re.M)
-          with open('pyproject.toml', 'w') as f:
-              f.write(new_content)
-          "
+          # The version has already been updated in create-nightly-tag job
+          # but we need to ensure we are on the correct tag/commit if needed.
+          # However, publish-pypi runs on the same branch by default.
+          # In the create-nightly-tag job we pushed a tag, here we should probably checkout that tag
+          # or just use the updated pyproject.toml if it's shared (it's not).
+          # So we need to checkout the tag.
+          git fetch origin "refs/tags/${{ needs.create-nightly-tag.outputs.nightly_version }}:refs/tags/${{ needs.create-nightly-tag.outputs.nightly_version }}"
+          git checkout ${{ needs.create-nightly-tag.outputs.nightly_version }}
 
       - name: Build wheel and source distribution
         run: |

scripts/ci/pypi_nightly_tag.py

@@ -0,0 +1,62 @@
+import sys
+import requests
+from packaging.version import Version
+from pathlib import Path
+import tomllib
+from typing import Optional
+
+PYPI_OPENRAG_NIGHTLY_URL = "https://pypi.org/pypi/openrag-nightly/json"
+PYPI_OPENRAG_URL = "https://pypi.org/pypi/openrag/json"
+
+def get_latest_published_version(is_nightly: bool) -> Optional[Version]:
+    url = PYPI_OPENRAG_NIGHTLY_URL if is_nightly else PYPI_OPENRAG_URL
+    res = requests.get(url, timeout=10)
+    if res.status_code == 404:
+        return None
+    res.raise_for_status()
+    try:
+        version_str = res.json()["info"]["version"]
+    except Exception as e:
+        msg = "Got unexpected response from PyPI"
+        raise RuntimeError(msg) from e
+    return Version(version_str)
+
+def create_tag():
+    # Read version from pyproject.toml
+    pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml"
+    with open(pyproject_path, "rb") as f:
+        pyproject_data = tomllib.load(f)
+
+    current_version_str = pyproject_data["project"]["version"]
+    current_version = Version(current_version_str)
+
+    try:
+        current_nightly_version = get_latest_published_version(is_nightly=True)
+    except (requests.RequestException, KeyError, ValueError):
+        current_nightly_version = None
+
+    build_number = "0"
+    latest_base_version = current_version.base_version
+    nightly_base_version = current_nightly_version.base_version if current_nightly_version else None
+
+    if latest_base_version == nightly_base_version:
+        dev_number = (current_nightly_version.dev if current_nightly_version.dev is not None else -1) if current_nightly_version else -1
+        build_number = str(dev_number + 1)
+
+    # Build PEP 440-compliant nightly version (without leading "v")
+    nightly_version_str = f"{latest_base_version}.dev{build_number}"
+
+    # Verify PEP440
+    Version(nightly_version_str)
+
+    # Git tag uses a leading "v" prefix
+    new_nightly_version = f"v{nightly_version_str}"
+    return new_nightly_version
+
+if __name__ == "__main__":
+    try:
+        tag = create_tag()
+        print(tag)
+    except Exception as e:
+        print(f"Error creating tag: {e}", file=sys.stderr)
+        sys.exit(1)

scripts/ci/update_pyproject_combined.py

@@ -0,0 +1,30 @@
+import sys
+from pathlib import Path
+
+# Add current dir to sys.path
+current_dir = Path(__file__).resolve().parent
+current_dir_str = str(current_dir)
+if current_dir_str not in sys.path:
+    sys.path.append(current_dir_str)
+
+from update_pyproject_name import update_pyproject_name
+from update_pyproject_version import update_version
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: update_pyproject_combined.py main <main_tag>")
+        sys.exit(1)
+
+    mode = sys.argv[1]
+    main_tag = sys.argv[2]
+
+    if mode != "main":
+        print("Only 'main' mode is supported")
+        sys.exit(1)
+
+    # Update name and version for openrag
+    update_pyproject_name("openrag-nightly")
+    update_version(main_tag)
+
+if __name__ == "__main__":
+    main()

scripts/ci/update_pyproject_name.py

@@ -0,0 +1,29 @@
+import re
+import sys
+from pathlib import Path
+
+def update_pyproject_name(new_name: str):
+    path = Path("pyproject.toml")
+    if not path.exists():
+        print("File pyproject.toml not found")
+        raise SystemExit(1)
+
+    content = path.read_text()
+    new_content = re.sub(r'^name = "[^"]+"', f'name = "{new_name}"', content, flags=re.M)
+
+    # Fail if the name pattern was not found / no substitution was made
+    if new_content == content:
+        print(
+            'Error: Could not find a line matching `name = "..."` in pyproject.toml to update.',
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    path.write_text(new_content)
+    print(f"Updated name in pyproject.toml to {new_name}")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: update_pyproject_name.py <new_name>")
+        sys.exit(1)
+    update_pyproject_name(sys.argv[1])

scripts/ci/update_pyproject_version.py

@@ -0,0 +1,48 @@
+import re
+import sys
+from pathlib import Path
+from packaging.version import Version, InvalidVersion
+
+def update_version(new_version):
+    pyproject_path = Path("pyproject.toml")
+    with open(pyproject_path, "r") as f:
+        content = f.read()
+
+    # Update the version field
+    # Removes 'v' prefix if present from tag
+    clean_version = new_version.lstrip('v')
+
+    # Validate that the resulting version is a valid PEP 440 version
+    try:
+        Version(clean_version)
+    except InvalidVersion:
+        print(
+            f"Error: '{clean_version}' is not a valid PEP 440 version after stripping any leading 'v'.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    new_content = re.sub(
+        r'^version = "[^"]+"',
+        f'version = "{clean_version}"',
+        content,
+        flags=re.M,
+    )
+
+    # Fail if the version pattern was not found / no substitution was made
+    if new_content == content:
+        print(
+            'Error: Could not find a line matching `version = "..."` in pyproject.toml to update.',
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    with open(pyproject_path, "w") as f:
+        f.write(new_content)
+    print(f"Updated pyproject.toml version to {clean_version}")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python update_pyproject_version.py <new_version>")
+        sys.exit(1)
+    update_version(sys.argv[1])

src/tui/utils/version_check.py

@@ -88,15 +88,18 @@ def get_current_version() -> str:
     Returns:
         Version string or "unknown" if not available
     """
-    try:
-        from importlib.metadata import version
-        return version("openrag")
-    except Exception:
+    for dist_name in ["openrag", "openrag-nightly"]:
         try:
-            from tui import __version__
-            return __version__
+            from importlib.metadata import version
+            return version(dist_name)
         except Exception:
-            return "unknown"
+            continue
+
+    try:
+        from tui import __version__
+        return __version__
+    except Exception:
+        return "unknown"
 
 
 def compare_versions(version1: str, version2: str) -> int:

src/utils/version_utils.py

@@ -8,7 +8,12 @@ def _get_openrag_version() -> str:
         from importlib.metadata import version, PackageNotFoundError
 
         try:
-            return version("openrag")
+            for dist_name in ["openrag", "openrag-nightly"]:
+                try:
+                    return version(dist_name)
+                except PackageNotFoundError:
+                    continue
+            raise PackageNotFoundError("openrag")
         except PackageNotFoundError:
             # Fallback: try to read from pyproject.toml if package not installed (dev mode)
             try: