From e49bfb08f7892adc010ad6a512b545e248f1cafd Mon Sep 17 00:00:00 2001 From: Zayd Date: Mon, 20 Oct 2025 16:52:08 -0700 Subject: [PATCH 1/8] Create publish_pypi.yaml --- .github/workflows/publish_pypi.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/publish_pypi.yaml diff --git a/.github/workflows/publish_pypi.yaml b/.github/workflows/publish_pypi.yaml new file mode 100644 index 0000000..ce07a20 --- /dev/null +++ b/.github/workflows/publish_pypi.yaml @@ -0,0 +1,17 @@ +name: Publish to Guardrails Hub + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - name: Build & Deploy + uses: guardrails-ai/guardrails/.github/actions/validator_pypi_publish@main + with: + guardrails_token: ${{ secrets.GR_GUARDRAILS_TOKEN }} + validator_id: mlcube/rag_retrieval From b05c03d29491c16a561ce6d818a70106ba0c5f90 Mon Sep 17 00:00:00 2001 From: Zayd Date: Tue, 21 Oct 2025 09:22:12 -0700 Subject: [PATCH 2/8] Update publish_pypi.yaml --- .github/workflows/publish_pypi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_pypi.yaml b/.github/workflows/publish_pypi.yaml index ce07a20..18574a2 100644 --- a/.github/workflows/publish_pypi.yaml +++ b/.github/workflows/publish_pypi.yaml @@ -13,5 +13,5 @@ jobs: - name: Build & Deploy uses: guardrails-ai/guardrails/.github/actions/validator_pypi_publish@main with: - guardrails_token: ${{ secrets.GR_GUARDRAILS_TOKEN }} + guardrails_token: ${{ secrets.PRIV_PYPI_PUBLISH_TOKEN }} validator_id: mlcube/rag_retrieval From 01e8b834423d5ec4da4aec2f0efd452e82734b6e Mon Sep 17 00:00:00 2001 From: Zayd Date: Wed, 22 Oct 2025 14:35:19 -0700 Subject: [PATCH 3/8] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ab59b90..867b1e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "rag-context-evaluator" -version = "0.0.1" +version = "0.0.2" description = "ML3 RAG Context Evaluator" authors = [ {name = "MLcube", email = "info@mlcube.com"} From 77fccadd0d0b5d2e73ff70266a2d49b4e36ed0c5 Mon Sep 17 00:00:00 2001 From: Caleb Courier <13314870+CalebCourier@users.noreply.github.com> Date: Thu, 23 Oct 2025 13:07:35 -0500 Subject: [PATCH 4/8] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d6141f2..93a4e95 100644 --- a/README.md +++ b/README.md @@ -161,8 +161,8 @@ Note: | `retrieved_context` | String | The context retrieved and used by the RAG system. | N/A (Required) | | `min_range_value` | String | The minimum value for the rating range used by the LLM judge. | 0 (the default of the validator class) | | `max_range_value` | String | The maximum value for the rating range used by the LLM judge. | 1 (the default of the validator class) | - + **Returns** -**`ValidationResult`**: Returns a `PassResult` if the LLM judge's rating meets or exceeds the pass threshold, or a `FailResult` with detailed explanation if the rating is below the threshold. \ No newline at end of file +**`ValidationResult`**: Returns a `PassResult` if the LLM judge's rating meets or exceeds the pass threshold, or a `FailResult` with detailed explanation if the rating is below the threshold. From 4e6e8a2fde9fc469f98f72ff4a9e6fb3a5700076 Mon Sep 17 00:00:00 2001 From: Caleb Courier <13314870+CalebCourier@users.noreply.github.com> Date: Thu, 23 Oct 2025 13:13:05 -0500 Subject: [PATCH 5/8] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 93a4e95..88042a0 100644 --- a/README.md +++ b/README.md @@ -153,8 +153,10 @@ Note: **Parameters** -- **`value`** _(Any)_: The input value to validate. -- **`metadata`** _(dict)_: A dictionary containing metadata required for validation. Keys and values must match the expectations of this validator. +- **`value`** *(Any)*: The input value to validate. +- **`metadata`** *(dict)*: A dictionary containing metadata required for validation. Keys and values must match the expectations of this validator. + + | Key | Type | Description | Default | | --- | --- | --- | --- | | `user_input` | String | The original user query passed into the RAG system. | N/A (Required) | From 97cb4093eafa958d329e915e432a7391d1d39078 Mon Sep 17 00:00:00 2001 From: Caleb Courier <13314870+CalebCourier@users.noreply.github.com> Date: Fri, 24 Oct 2025 09:30:23 -0500 Subject: [PATCH 6/8] Remove publish script from main and fix install in pr_qc workflow (#2) * remove CI from main * re-add qc * lint fixes * use uv in qc workflow * debug uv in qc * more debug * use python from uv * wrap quotes * more debug logs * try using virtual environment * cleanup * more cleanup --- .github/workflows/pr_qc.yml | 8 +++++--- .github/workflows/publish_pypi.yaml | 17 ----------------- uv.lock | 4 ++-- validator/main.py | 2 -- 4 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 .github/workflows/publish_pypi.yaml diff --git a/.github/workflows/pr_qc.yml b/.github/workflows/pr_qc.yml index 7a46efe..f1ce7b8 100644 --- a/.github/workflows/pr_qc.yml +++ b/.github/workflows/pr_qc.yml @@ -9,11 +9,13 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v7 with: python-version: 3.11 - name: Run qa run: | - pip install ".[dev]" + python -m venv ./.venv + source ./.venv/bin/activate + make dev make qa diff --git a/.github/workflows/publish_pypi.yaml b/.github/workflows/publish_pypi.yaml deleted file mode 100644 index 18574a2..0000000 --- a/.github/workflows/publish_pypi.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Publish to Guardrails Hub - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - setup: - runs-on: ubuntu-latest - steps: - - name: Build & Deploy - uses: guardrails-ai/guardrails/.github/actions/validator_pypi_publish@main - with: - guardrails_token: ${{ secrets.PRIV_PYPI_PUBLISH_TOKEN }} - validator_id: mlcube/rag_retrieval diff --git a/uv.lock b/uv.lock index be1b535..b4ccf02 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10, <=3.13" resolution-markers = [ "python_full_version >= '3.13'", @@ -1959,7 +1959,7 @@ wheels = [ [[package]] name = "rag-context-evaluator" -version = "0.0.1" +version = "0.0.2" source = { virtual = "." } dependencies = [ { name = "guardrails-ai" }, diff --git a/validator/main.py b/validator/main.py index 703e0a3..c3344b5 100644 --- a/validator/main.py +++ b/validator/main.py @@ -1,6 +1,4 @@ from typing import Any, Callable, Dict, Optional -from guardrails import Guard -import openai import logging from guardrails.validator_base import ( FailResult, From c4909d1f6d2036e4779efa52f30dbba4673fd712 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Wed, 29 Oct 2025 12:14:48 -0500 Subject: [PATCH 7/8] add env var to qa for tests to work --- .github/workflows/pr_qc.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr_qc.yml b/.github/workflows/pr_qc.yml index f1ce7b8..799e5fd 100644 --- a/.github/workflows/pr_qc.yml +++ b/.github/workflows/pr_qc.yml @@ -14,6 +14,8 @@ jobs: with: python-version: 3.11 - name: Run qa + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | python -m venv ./.venv source ./.venv/bin/activate From eedaaa638810658be4493e16b5e0c6249a2b0f81 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Wed, 29 Oct 2025 12:24:54 -0500 Subject: [PATCH 8/8] remove cache --- .github/workflows/pr_qc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr_qc.yml b/.github/workflows/pr_qc.yml index 799e5fd..f7f4908 100644 --- a/.github/workflows/pr_qc.yml +++ b/.github/workflows/pr_qc.yml @@ -12,6 +12,7 @@ jobs: - name: Install the latest version of uv uses: astral-sh/setup-uv@v7 with: + enable-cache: false python-version: 3.11 - name: Run qa env: