Skip to content

feat(training): update evals API & example notebooks #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bartolomej wants to merge 3 commits into
base: feat/military-strikes-notebook
Choose a base branch
from
Open

feat(training): update evals API & example notebooks #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c613fb7
update api client
bartolomej Mar 27, 2026
b6f2975
update millitary strikes notebook
bartolomej Mar 27, 2026
74844ce
regenerate client
bartolomej Mar 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
666 changes: 331 additions & 335 deletions notebooks/fine_tuning/04_military_strikes.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

263 changes: 66 additions & 197 deletions openapi/openapi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,6 @@
"description": "Generate verified, grounded datasets at scale for LLM fine-tuning and evaluation",
"version": "1.0.0"
},
"servers": [
{
"url": "/api/public/v1"
}
],
"paths": {
"/openai/chat/completions": {
"post": {
Expand Down Expand Up @@ -2221,20 +2216,12 @@
"dataset": {
"$ref": "#/components/schemas/SampleDatasetConfig"
},
"model_id": {
"type": "string",
"title": "Model Id"
},
"benchmark_model_id": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Benchmark Model Id"
"models": {
"items": {
"$ref": "#/components/schemas/EvalModel"
},
"type": "array",
"title": "Models"
},
"temperature": {
"type": "number",
Expand All @@ -2245,7 +2232,7 @@
"type": "object",
"required": [
"dataset",
"model_id"
"models"
],
"title": "CreateEvalJobRequest"
},
Expand Down Expand Up @@ -2489,12 +2476,6 @@
{
"$ref": "#/components/schemas/NewsSeedGenerator"
},
{
"$ref": "#/components/schemas/TopicTreeSeedGenerator"
},
{
"$ref": "#/components/schemas/CsvSeedGenerator"
},
{
"$ref": "#/components/schemas/QuestionAndLabelGenerator"
},
Expand All @@ -2514,7 +2495,6 @@
"discriminator": {
"propertyName": "config_type",
"mapping": {
"CSV_SEED_GENERATOR": "#/components/schemas/CsvSeedGenerator",
"FILESET_QUERY_SEED_GENERATOR": "#/components/schemas/FileSetQuerySeedGenerator",
"FILESET_SEED_GENERATOR": "#/components/schemas/FileSetSeedGenerator",
"FORWARD_LOOKING_QUESTION_GENERATOR": "#/components/schemas/ForwardLookingQuestionGenerator",
Expand All @@ -2524,7 +2504,6 @@
"QUESTION_GENERATOR": "#/components/schemas/QuestionGenerator",
"QUESTION_PIPELINE": "#/components/schemas/QuestionPipeline",
"QUESTION_RENDERER": "#/components/schemas/QuestionRenderer",
"TOPIC_TREE_SEED_GENERATOR": "#/components/schemas/TopicTreeSeedGenerator",
"WEB_SEARCH_LABELER": "#/components/schemas/WebSearchLabeler"
}
}
Expand Down Expand Up @@ -2596,73 +2575,6 @@
],
"title": "CreateTransformJobRequest"
},
"CsvSeedGenerator": {
"properties": {
"config_type": {
"type": "string",
"const": "CSV_SEED_GENERATOR",
"title": "Config Type",
"description": "Type of transform configuration",
"default": "CSV_SEED_GENERATOR"
},
"file_id": {
"anyOf": [
{
"type": "string"
},
{
"items": {
"type": "string"
},
"type": "array"
}
],
"title": "File Id",
"description": "OrgFile ID(s) from POST /files/ upload response"
},
"seed_text_column": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Seed Text Column",
"description": "Column name for Seed.seed_text; if None, serialize entire row as JSON"
},
"label_column": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Label Column",
"description": "Column name for pre-existing labels (populates Sample.label)"
},
"date_column": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Date Column",
"description": "Column name for Seed.seed_creation_date"
}
},
"type": "object",
"required": [
"file_id"
],
"title": "CsvSeedGenerator"
},
"DatasetMetadata": {
"properties": {
"id": {
Expand Down Expand Up @@ -2833,24 +2745,16 @@
"type": "string",
"title": "Organization Id"
},
"model_id": {
"type": "string",
"title": "Model Id"
"models": {
"items": {
"$ref": "#/components/schemas/EvalModel"
},
"type": "array",
"title": "Models"
},
"dataset": {
"$ref": "#/components/schemas/SampleDatasetConfig"
},
"benchmark_model_id": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Benchmark Model Id"
},
"temperature": {
"type": "number",
"title": "Temperature",
Expand All @@ -2870,7 +2774,7 @@
"type": "object",
"required": [
"organization_id",
"model_id",
"models",
"dataset"
],
"title": "EvalConfig"
Expand Down Expand Up @@ -3000,6 +2904,30 @@
],
"title": "EvalJobStatus"
},
"EvalModel": {
"properties": {
"model_id": {
"type": "string",
"title": "Model Id"
},
"label": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Label"
}
},
"type": "object",
"required": [
"model_id"
],
"title": "EvalModel"
},
"EventUsageSummary": {
"properties": {
"count": {
Expand Down Expand Up @@ -5285,12 +5213,6 @@
{
"$ref": "#/components/schemas/FileSetQuerySeedGenerator"
},
{
"$ref": "#/components/schemas/TopicTreeSeedGenerator"
},
{
"$ref": "#/components/schemas/CsvSeedGenerator"
},
{
"$ref": "#/components/schemas/MockTransformConfig"
}
Expand All @@ -5299,13 +5221,11 @@
"propertyName": "config_type",
"mapping": {
"BIGQUERY_SEED_GENERATOR": "#/components/schemas/BigQuerySeedGenerator",
"CSV_SEED_GENERATOR": "#/components/schemas/CsvSeedGenerator",
"FILESET_QUERY_SEED_GENERATOR": "#/components/schemas/FileSetQuerySeedGenerator",
"FILESET_SEED_GENERATOR": "#/components/schemas/FileSetSeedGenerator",
"GDELT_SEED_GENERATOR": "#/components/schemas/GdeltSeedGenerator",
"MOCK": "#/components/schemas/MockTransformConfig",
"NEWS_SEED_GENERATOR": "#/components/schemas/NewsSeedGenerator",
"TOPIC_TREE_SEED_GENERATOR": "#/components/schemas/TopicTreeSeedGenerator"
"NEWS_SEED_GENERATOR": "#/components/schemas/NewsSeedGenerator"
}
}
},
Expand Down Expand Up @@ -6084,72 +6004,6 @@
"title": "TemporalConstraint",
"description": "Temporal filtering direction relative to the seed document's date.\n\nUses the `file_date` metadata key (unix timestamp int) stored on each\nGemini document by fileset_file_processor.\n\nBEFORE: file_date <= seed_timestamp (context: no lookahead bias)\nAFTER: file_date > seed_timestamp (labels: find resolutions)"
},
"TopicTreeSeedGenerator": {
"properties": {
"config_type": {
"type": "string",
"const": "TOPIC_TREE_SEED_GENERATOR",
"title": "Config Type",
"description": "Type of transform configuration",
"default": "TOPIC_TREE_SEED_GENERATOR"
},
"topic": {
"anyOf": [
{
"type": "string"
},
{
"items": {
"type": "string"
},
"type": "array"
}
],
"title": "Topic",
"description": "Root topic(s) to decompose into subtopic trees"
},
"tree_depth": {
"type": "integer",
"maximum": 10.0,
"minimum": 1.0,
"title": "Tree Depth",
"description": "Levels of recursive expansion",
"default": 2
},
"tree_degree": {
"type": "integer",
"maximum": 20.0,
"minimum": 2.0,
"title": "Tree Degree",
"description": "Subtopics generated per node",
"default": 5
},
"model_name": {
"type": "string",
"title": "Model Name",
"description": "LLM model for subtopic generation",
"default": "google/gemini-3-flash-preview"
},
"model_system_prompt": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Model System Prompt",
"description": "Optional system prompt for the LLM"
}
},
"type": "object",
"required": [
"topic"
],
"title": "TopicTreeSeedGenerator",
"description": "Generates diverse seeds by recursively decomposing broad topics into specific subtopics.\n\nGiven one or more root topics, uses an LLM to break each into `tree_degree` subtopics,\nthen repeats `tree_depth` levels deep. The leaf paths become seeds for downstream transforms.\nThis produces tree_degree^tree_depth seeds per root topic, each more specific than the root.\n\nExample:\n TopicTreeSeedGenerator(topic=\"AI Regulation\", tree_depth=2, tree_degree=4)\n # Produces 16 specific seeds like \"AI Regulation \u2192 Healthcare \u2192 FDA approval of\n # diagnostic algorithms\" that feed into QuestionGenerator or other downstream transforms."
},
"TrainingConfig": {
"properties": {
"dataset": {
Expand Down Expand Up @@ -6309,6 +6163,20 @@
],
"title": "Model Id"
},
"model_id_by_step": {
"anyOf": [
{
"additionalProperties": {
"type": "string"
},
"type": "object"
},
{
"type": "null"
}
],
"title": "Model Id By Step"
},
"reward_history": {
"anyOf": [
{
Expand Down Expand Up @@ -6356,17 +6224,6 @@
],
"title": "Cost Dollars"
},
"dataset_hf_repo": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Dataset Hf Repo"
},
"error_message": {
"anyOf": [
{
Expand Down Expand Up @@ -6799,6 +6656,13 @@
"type": {
"type": "string",
"title": "Error Type"
},
"input": {
"title": "Input"
},
"ctx": {
"type": "object",
"title": "Context"
}
},
"type": "object",
Expand Down Expand Up @@ -6875,5 +6739,10 @@
"scheme": "bearer"
}
}
}
},
"servers": [
{
"url": "/api/public/v1"
}
]
}
Loading