Merge pull request #656 from Labelbox/mmw/create-model-run-with-config

Add model run config on create [AL-3060]

Author: mmarinwilcke

CHANGELOG.md

@@ -654,3 +654,7 @@ a `Label`. Default value is 0.0.
 
 ## Version 2.2 (2019-10-18)
 Changelog not maintained before version 2.2.
+
+### Changed
+* `Model.create_model_run()`
+    * Add training metadata config as a model run creation param

examples/model_diagnostics/model_diagnostics_guide.ipynb

@@ -350,7 +350,8 @@
    "source": [
     "lb_model = client.create_model(name=f\"{project.name}-model\",\n",
     "                               ontology_id=project.ontology().uid)\n",
-    "lb_model_run = lb_model.create_model_run(\"0.0.0\")\n",
+    "lb_model_run_hyperparameters = {\"batch_size\": 1000}\n",
+    "lb_model_run = lb_model.create_model_run(\"0.0.0\", lb_model_run_hyperparameters)\n",
     "lb_model_run.upsert_labels([label.uid for label in labels])"
    ]
   },

labelbox/schema/model.py

@@ -18,23 +18,26 @@ class Model(DbObject):
     name = Field.String("name")
     model_runs = Relationship.ToMany("ModelRun", False)
 
-    def create_model_run(self, name) -> "ModelRun":
+    def create_model_run(self, name, config=None) -> "ModelRun":
         """ Creates a model run belonging to this model.
 
         Args:
             name (string): The name for the model run.
+            config (json): Model run's training metadata config
         Returns:
             ModelRun, the created model run.
         """
         name_param = "name"
+        config_param = "config"
         model_id_param = "modelId"
         ModelRun = Entity.ModelRun
-        query_str = """mutation CreateModelRunPyApi($%s: String!, $%s: ID!) {
-            createModelRun(data: {name: $%s, modelId: $%s}) {%s}}""" % (
-            name_param, model_id_param, name_param, model_id_param,
-            query.results_query_part(ModelRun))
+        query_str = """mutation CreateModelRunPyApi($%s: String!, $%s: Json, $%s: ID!) {
+            createModelRun(data: {name: $%s, trainingMetadata: $%s, modelId: $%s}) {%s}}""" % (
+            name_param, config_param, model_id_param, name_param, config_param,
+            model_id_param, query.results_query_part(ModelRun))
         res = self.client.execute(query_str, {
             name_param: name,
+            config_param: config,
             model_id_param: self.uid
         })
         return ModelRun(self.client, res["createModelRun"])

labelbox/schema/model_run.py

@@ -32,6 +32,7 @@ class ModelRun(DbObject):
     created_at = Field.DateTime("created_at")
     created_by_id = Field.String("created_by_id", "createdBy")
     model_id = Field.String("model_id")
+    training_metadata = Field.Json("training_metadata")
 
     class Status(Enum):
         EXPORTING_DATA = "EXPORTING_DATA"

tests/integration/annotation_import/test_model_run.py

@@ -14,8 +14,11 @@ def test_model_run(client, configured_project_with_label, rand_gen):
     model = client.create_model(data["name"], data["ontology_id"])
 
     name = rand_gen(str)
-    model_run = model.create_model_run(name)
+    config = {"batch_size": 100, "reruns": None}
+    model_run = model.create_model_run(name, config)
     assert model_run.name == name
+    assert model_run.training_metadata["batchSize"] == config["batch_size"]
+    assert model_run.training_metadata["reruns"] == config["reruns"]
     assert model_run.model_id == model.uid
     assert model_run.created_by_id == client.get_user().uid
 
@@ -32,6 +35,12 @@ def test_model_run(client, configured_project_with_label, rand_gen):
     assert fetch_model_run == model_run
 
 
+def test_model_run_no_config(rand_gen, model):
+    name = rand_gen(str)
+    model_run = model.create_model_run(name)
+    assert model_run.name == name
+
+
 def test_model_run_delete(client, model_run):
     models_before = list(client.get_models())
     model_before = models_before[0]