Skip to content

T3C-433 Evaluate GPT-5-mini #283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

T3C-433 Evaluate GPT-5-mini #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a21ded9
Update configs for chatgpt-5
juggler434 Sep 5, 2025
d3a7152
Adds in support for gpt-5-mini
juggler434 Sep 10, 2025
fb0d5b4
try adjusting reasoning effort
juggler434 Oct 20, 2025
e1d3ac4
removes temperature from llm calls
juggler434 Oct 21, 2025
564030c
updates reasoning effort to low
juggler434 Oct 21, 2025
bf13394
Updates reasoning effort to be minimal
juggler434 Oct 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion express-server/src/routes/create.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ const buildPipelineJob = (
cruxes: updatedConfig.cruxesEnabled,
},
llm: {
model: "gpt-4o-mini", // ! Change when we allow different models
model: "gpt-5-mini", // ! Change when we allow different models
},
},
data: updatedConfig.data,
Expand Down
4 changes: 3 additions & 1 deletion pyserver/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
DRY_RUN = False

# cheapest for testing
MODEL = "gpt-4o-mini" # prod default: "gpt-4-turbo-preview"
MODEL = "gpt-5-mini" # prod default: "gpt-4-turbo-preview"

COST_BY_MODEL = {
# GPT-4o mini: Input is $0.150 / 1M tokens, Output is $0.600 / 1M tokens
Expand All @@ -17,6 +17,8 @@
# GPT-4o : Input is $2.50 / 1M tokens, Output is $10.00/1M tokens
# or: input is $0.0025/1K tokens, output is $0.01/1K tokens
"gpt-4o": {"in_per_1K": 0.0025, "out_per_1K": 0.01},
# GPT-5-mini: Input is $0.250 / 1M tokens, Output is $2.00/1M tokens
"gpt-5-mini": {"in_per_1K": 0.00025, "out_per_1K": 0.002}
}

# for web-app mode, require at least 3 words in order to extract meaningful claims
Expand Down
12 changes: 5 additions & 7 deletions pyserver/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,8 +499,8 @@ def comments_to_tree(
{"role": "system", "content": req.llm.system_prompt},
{"role": "user", "content": full_prompt},
],
temperature=0.0,
Copy link
Collaborator Author

@juggler434 juggler434 Sep 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This parameter is no longer supported by gpt-5

response_format={"type": "json_object"},
reasoning_effort="minimal",
)
try:
tree = json.loads(response.choices[0].message.content)
Expand Down Expand Up @@ -634,7 +634,6 @@ def comment_to_claims(llm: dict, comment: str, tree: dict, api_key: str, comment
{"role": "system", "content": llm.system_prompt},
{"role": "user", "content": full_prompt},
],
temperature=0.0,
response_format=ClaimsSchema,
)

Expand All @@ -652,7 +651,6 @@ def comment_to_claims(llm: dict, comment: str, tree: dict, api_key: str, comment
llm_metadata = {
"model": llm.model_name,
"prompt_version": "v1.0", # Update when prompt changes
"temperature": getattr(llm, 'temperature', None),
}
audit_logger.log_claims_extraction_empty(
comment_id,
Expand Down Expand Up @@ -684,7 +682,6 @@ def comment_to_claims(llm: dict, comment: str, tree: dict, api_key: str, comment
llm_metadata = {
"model": llm.model_name,
"prompt_version": "v1.0", # Update when prompt changes
"temperature": getattr(llm, 'temperature', None),
}

# For now, claim_ids will be the same as comment_id since we don't have unique claim IDs yet
Expand Down Expand Up @@ -1146,8 +1143,8 @@ def dedup_claims(claims: list, llm: LLMConfig, api_key: str, topic_name: str = "
{"role": "system", "content": llm.system_prompt},
{"role": "user", "content": full_prompt},
],
temperature=0.0,
response_format={"type": "json_object"},
reasoning_effort="minimal",
)
try:
deduped_claims = response.choices[0].message.content
Expand Down Expand Up @@ -1723,8 +1720,8 @@ def generate_topic_summaries(
{"role": "system", "content": llm_config["system_prompt"]},
{"role": "user", "content": full_prompt},
],
temperature=0.0,
response_format={"type": "json_object"},
reasoning_effort="minimal"
)

try:
Expand Down Expand Up @@ -1888,8 +1885,9 @@ def cruxes_for_topic(
{"role": "system", "content": llm.system_prompt},
{"role": "user", "content": full_prompt},
],
temperature=0.0,
response_format={"type": "json_object"},
reasoning_effort="minimal",

)
crux = response.choices[0].message.content
try:
Expand Down
2 changes: 1 addition & 1 deletion pyserver/test_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
# NOTE: gpt-4o-mini is cheaper/better for basic tests, but it fails on some very basic deduplication
API_KEY = os.getenv('OPENAI_API_KEY')
base_llm = {
"model_name" : "gpt-4o-mini",
"model_name" : "gpt-5-mini",
"system_prompt": config.SYSTEM_PROMPT,
"api_key" : API_KEY
}
Expand Down
1 change: 1 addition & 0 deletions pyserver/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def token_cost(model_name:str, tok_in:int, tok_out:int):
if model_name not in config.COST_BY_MODEL:
print("model undefined!")
return -1

return 0.001 * (tok_in * config.COST_BY_MODEL[model_name]["in_per_1K"] + tok_out * config.COST_BY_MODEL[model_name]["out_per_1K"])

def cute_print(json_obj):
Expand Down
Loading