Skip to content

[SC-9395] Add RAG benchmarking demo notebook #346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juanmleng merged 8 commits into from
Apr 9, 2025
Merged

[SC-9395] Add RAG benchmarking demo notebook #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8162f0e
Add rag notebook to showcase benchmarking using comparison tests
juanmleng Apr 2, 2025
c71be4b
Update text in notebook
juanmleng Apr 3, 2025
c4bd1a5
Merge branch 'main' into juan/sc-9395/extend-llm-testing-in-rag-use-case
juanmleng Apr 3, 2025
178c5e2
Add title to response relevancy figures
juanmleng Apr 3, 2025
4f9f742
Constrained ragas to a working version
juanmleng Apr 3, 2025
37da615
Update lock
juanmleng Apr 3, 2025
bc1f039
2.8.18
juanmleng Apr 9, 2025
6751e8a
Add validmind install with LLM support
juanmleng Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,635 changes: 1,635 additions & 0 deletions notebooks/code_samples/nlp_and_llm/rag_benchmark_demo.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

816 changes: 461 additions & 355 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ description = "ValidMind Library"
license = "Commercial License"
name = "validmind"
readme = "README.pypi.md"
version = "2.8.17"
version = "2.8.18"

[tool.poetry.dependencies]
aiohttp = {extras = ["speedups"], version = "*"}
Expand Down Expand Up @@ -39,7 +39,7 @@ polars = "*"
pycocoevalcap = {version = "^1.2", optional = true}
python = ">=3.8.1,<3.12"
python-dotenv = "*"
ragas = {version = ">=0.2.3", optional = true}
ragas = {version = ">=0.2.3,<=0.2.7", optional = true}
rouge = ">=1"
scikit-learn = "*,<1.6.0"
scipy = "*"
Expand Down
2 changes: 1 addition & 1 deletion validmind/__version__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "2.8.17"
__version__ = "2.8.18"
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/AnswerCorrectness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,10 @@ def AnswerCorrectness(

score_column = "answer_correctness"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Answer Correctness"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Answer Correctness")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/ContextEntityRecall.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,10 @@ def ContextEntityRecall(

score_column = "context_entity_recall"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Context Entity Recall"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Context Entity Recall")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/ContextPrecision.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ def ContextPrecision(

score_column = "llm_context_precision_with_reference"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,10 @@ def ContextPrecisionWithoutReference(

score_column = "llm_context_precision_without_reference"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Context Precision"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Context Precision")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/ContextRecall.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ def ContextRecall(

score_column = "context_recall"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Context Recall"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Context Recall")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/Faithfulness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,10 @@ def Faithfulness(

score_column = "faithfulness"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Faithfulness"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Faithfulness")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/ResponseRelevancy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,10 @@ def ResponseRelevancy(

score_column = "answer_relevancy"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Response Relevancy"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Response Relevancy")

return (
{
Expand Down
6 changes: 4 additions & 2 deletions validmind/tests/model_validation/ragas/SemanticSimilarity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,10 @@ def SemanticSimilarity(

score_column = "semantic_similarity"

fig_histogram = px.histogram(x=result_df[score_column].to_list(), nbins=10)
fig_box = px.box(x=result_df[score_column].to_list())
fig_histogram = px.histogram(
x=result_df[score_column].to_list(), nbins=10, title="Semantic Similarity"
)
fig_box = px.box(x=result_df[score_column].to_list(), title="Semantic Similarity")

return (
{
Expand Down