From 28f7e7fb2a6f10e4ff5fb49cb36a3a9cd320f3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:21:24 -0500 Subject: [PATCH 01/52] Add Data Preparator cookiecutter template --- .../data_preparator_mlcube/coookiecutter.json | 9 ++++ .../hooks/post_gen_project.py | 13 ++++++ .../mlcube/mlcube.yaml | 33 ++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++ .../project/mlcube.py | 44 +++++++++++++++++++ 6 files changed, 130 insertions(+) create mode 100644 templates/data_preparator_mlcube/coookiecutter.json create mode 100644 templates/data_preparator_mlcube/hooks/post_gen_project.py create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py diff --git a/templates/data_preparator_mlcube/coookiecutter.json b/templates/data_preparator_mlcube/coookiecutter.json new file mode 100644 index 000000000..d33b56e1e --- /dev/null +++ b/templates/data_preparator_mlcube/coookiecutter.json @@ -0,0 +1,9 @@ +{ + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "mlcube_name": "Data Preparator MLCube", + "description": "Data Preparator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest", + "use_separate_output_labels": "n" +} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..aa5bd6fa3 --- /dev/null +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +import os +import shutil + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + if "{{ cookiecutter.use_separate_output_labels }}" != "y": + input_labels_path = os.path.join( + "{{ cookiecutter.project_slug }}", "mlcube/workspace/input_labels" + ) + shutil.rmtree() diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml new file mode 100644 index 000000000..38be9e0ce --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml @@ -0,0 +1,33 @@ +name: {{ cookiecutter.mlcube_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + prepare: + parameters: + inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + outputs: { + output_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% endif %} + } + sanity_check: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + statistics: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py new file mode 100644 index 000000000..7b38614c9 --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py @@ -0,0 +1,44 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("prepare") +def prepare( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: str = typer.Option(..., "--output_labels_path"), + {% endif %} +): + # Modify the prepare command as needed + raise NotImplementedError("The prepare method is not yet implemented") + + +@app.command("sanity_check") +def sanity_check( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), +): + # Modify the sanity_check command as needed + raise NotImplementedError("The sanity check method is not yet implemented") + + +@app.command("statistics") +def sanity_check( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + out_path: str = typer.Option(..., "--output_path"), +): + # Modify the statistics command as needed + raise NotImplementedError("The statistics method is not yet implemented") + + +if __name__ == "__main__": + app() From 6f9e19e3319ba45066156dedc7007d8b432139bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:35:49 -0500 Subject: [PATCH 02/52] Rename cookiecutter folder --- .../mlcube/mlcube.yaml | 0 .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 0 .../project/mlcube.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/mlcube/mlcube.yaml (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/mlcube/workspace/parameters.yaml (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/project/Dockerfile (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/project/mlcube.py (100%) diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py From df6e6a2463c7afaec2e85a75a186d9f079538d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:40:43 -0500 Subject: [PATCH 03/52] Temporarily remove possibly offending files --- .../mlcube/mlcube.yaml | 33 ------------------- .../mlcube/workspace/parameters.yaml | 0 2 files changed, 33 deletions(-) delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index 38be9e0ce..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: {{ cookiecutter.mlcube_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - prepare: - parameters: - inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} - outputs: { - output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ - {% endif %} - } - sanity_check: - parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - statistics: - parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 From a7db0cfe8ff8f7074b31e6a4e32f292f85ac89a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:41:24 -0500 Subject: [PATCH 04/52] Remove cookicutter conditionals --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..fcdb17207 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,7 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From a7a6d15a51355a0d3dca0dee9e44ba181cfe2bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:42:29 -0500 Subject: [PATCH 05/52] Inclube back missing pieces of template --- .../mlcube/mlcube.yaml | 33 +++++++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/mlcube.py | 2 ++ 3 files changed, 35 insertions(+) create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..38be9e0ce --- /dev/null +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,33 @@ +name: {{ cookiecutter.mlcube_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + prepare: + parameters: + inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + outputs: { + output_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% endif %} + } + sanity_check: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + statistics: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index fcdb17207..7b38614c9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,7 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), + {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From e2f7108dca68182f4241e8b2ceaf5b01d18b6a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:43:30 -0500 Subject: [PATCH 06/52] remove cookiecutter typo --- .../{coookiecutter.json => cookiecutter.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename templates/data_preparator_mlcube/{coookiecutter.json => cookiecutter.json} (100%) diff --git a/templates/data_preparator_mlcube/coookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json similarity index 100% rename from templates/data_preparator_mlcube/coookiecutter.json rename to templates/data_preparator_mlcube/cookiecutter.json From 581b5bb1e0cfabaa3c3a20b84d5aa2c4d90e0393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:45:09 -0500 Subject: [PATCH 07/52] Use project_name attribute --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index d33b56e1e..1bba774b1 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -1,6 +1,6 @@ { "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "mlcube_name": "Data Preparator MLCube", + "project_name": "Data Preparator MLCube", "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 38be9e0ce..a617061b9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -1,4 +1,4 @@ -name: {{ cookiecutter.mlcube_name }} +name: {{ cookiecutter.project_name }} description: {{ cookiecutter.description }} authors: - {name: {{ cookiecutter.author_name }}} From fd778041687e5e5243c213bc7d2e5b7200413d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:48:05 -0500 Subject: [PATCH 08/52] Change cookiecutter fields order --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index 1bba774b1..95b9a12d5 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -1,6 +1,6 @@ { - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", "project_name": "Data Preparator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", From 6eebd59060a12a258ae165dfbf2e20a056afe102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:54:04 -0500 Subject: [PATCH 09/52] Create empty directories on hook --- .../hooks/post_gen_project.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index aa5bd6fa3..82dc878ae 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -1,13 +1,21 @@ #!/usr/bin/env python import os -import shutil PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) if __name__ == "__main__": - if "{{ cookiecutter.use_separate_output_labels }}" != "y": - input_labels_path = os.path.join( - "{{ cookiecutter.project_slug }}", "mlcube/workspace/input_labels" - ) - shutil.rmtree() + dir = "{{ cookiecutter.project_slug }}" + + input_data_path = os.path.join(dir, "mlcube/workspace/input_data") + data_path = os.path.join(dir, "mlcube/workspace/data") + labels_path = os.path.join(dir, "mlcube/workspace/labels") + + paths = [input_data_path, data_path, labels_path] + + if "{{ cookiecutter.use_separate_output_labels }}" == "y": + input_labels_path = os.path.join(dir, "mlcube/workspace/input_labels") + paths.append(input_labels_path) + + for path in paths: + os.makedirs(path, exist_ok=True) From 5ef86a2d3c9b1f78d6eaa49cc6e6be8f1e8df64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:56:57 -0500 Subject: [PATCH 10/52] Fix empty folders paths --- .../data_preparator_mlcube/hooks/post_gen_project.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index 82dc878ae..540b83bd7 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -5,16 +5,14 @@ if __name__ == "__main__": - dir = "{{ cookiecutter.project_slug }}" - - input_data_path = os.path.join(dir, "mlcube/workspace/input_data") - data_path = os.path.join(dir, "mlcube/workspace/data") - labels_path = os.path.join(dir, "mlcube/workspace/labels") + input_data_path = "mlcube/workspace/input_data" + data_path = "mlcube/workspace/data" + labels_path = "mlcube/workspace/labels" paths = [input_data_path, data_path, labels_path] if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = os.path.join(dir, "mlcube/workspace/input_labels") + input_labels_path = "mlcube/workspace/input_labels" paths.append(input_labels_path) for path in paths: From d04baf849e22058d69e1d5fd3eac55edafdb55a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:05:54 -0500 Subject: [PATCH 11/52] Create evaluator mlcube cookiecutter template --- templates/evaluator_mlcube/cookiecutter.json | 8 +++++ .../hooks/post_gen_project.py | 18 +++++++++++ .../mlcube/mlcube.yaml | 22 +++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++++++++ .../project/mlcube.py | 26 ++++++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 templates/evaluator_mlcube/cookiecutter.json create mode 100644 templates/evaluator_mlcube/hooks/post_gen_project.py create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json new file mode 100644 index 000000000..eed0126ab --- /dev/null +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -0,0 +1,8 @@ +{ + "project_name": "Data Preparator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "description": "Data Preparator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest", +} \ No newline at end of file diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..74c69c0d7 --- /dev/null +++ b/templates/evaluator_mlcube/hooks/post_gen_project.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import os + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + preds_path = "mlcube/workspace/predictions" + labels_path = "mlcube/workspace/labels" + + paths = [preds_path, labels_path] + + if "{{ cookiecutter.use_separate_output_labels }}" == "y": + input_labels_path = "mlcube/workspace/input_labels" + paths.append(input_labels_path) + + for path in paths: + os.makedirs(path, exist_ok=True) diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..c486308c9 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,22 @@ +name: {{ cookiecutter.project_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + evaluate: + # Computes evaluation metrics on the given predictions and ground truths + parameters: + inputs: {predictions: predictions, labels: labels, parameters_file: parameters.yaml} + outputs: {output_path: {type: "file", default: "results.yaml"}} \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py new file mode 100644 index 000000000..1de721eb9 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -0,0 +1,26 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("evaluate") +def prepare( + labels: str = typer.Option(..., "--labels"), + predictions: str = typer.Option(..., "--predictions"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), +): + # Modify the prepare command as needed + raise NotImplementedError("The evaluate method is not yet implemented") + + +@app.command("hotfix") +def hotfix(): + # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY + pass + + +if __name__ == "__main__": + app() From 02cec0166faeb4b63522cbd5190bdd2cb950d49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:06:36 -0500 Subject: [PATCH 12/52] Fix JSON Syntax Error --- templates/evaluator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json index eed0126ab..d716638f2 100644 --- a/templates/evaluator_mlcube/cookiecutter.json +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -4,5 +4,5 @@ "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", - "docker_image_name": "docker/image:latest", + "docker_image_name": "docker/image:latest" } \ No newline at end of file From b3d7a1d85b849f29cfbd5f6c7dde4006af3c188f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:09:03 -0500 Subject: [PATCH 13/52] Update template default values --- templates/evaluator_mlcube/cookiecutter.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json index d716638f2..07cd7e295 100644 --- a/templates/evaluator_mlcube/cookiecutter.json +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -1,7 +1,7 @@ { - "project_name": "Data Preparator MLCube", + "project_name": "Evaluator MLCube", "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Data Preparator MLCube Template. Provided by MLCommons", + "description": "Evaluator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest" From 7338236411635e7581008b25b6465f5055d50e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:09:20 -0500 Subject: [PATCH 14/52] Remove reference to undefined template variable --- templates/evaluator_mlcube/hooks/post_gen_project.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py index 74c69c0d7..e36bdaef0 100644 --- a/templates/evaluator_mlcube/hooks/post_gen_project.py +++ b/templates/evaluator_mlcube/hooks/post_gen_project.py @@ -9,10 +9,5 @@ labels_path = "mlcube/workspace/labels" paths = [preds_path, labels_path] - - if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = "mlcube/workspace/input_labels" - paths.append(input_labels_path) - for path in paths: os.makedirs(path, exist_ok=True) From d1cec5e668c1d1fa312b0abf7e36cca30a07ad2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:23:11 -0500 Subject: [PATCH 15/52] Implement model mlcube cookiecutter template --- templates/model_mlcube/cookiecutter.json | 8 +++++ .../model_mlcube/hooks/post_gen_project.py | 13 ++++++++ .../mlcube/mlcube.yaml | 29 +++++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++++++++ .../project/mlcube.py | 28 +++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 templates/model_mlcube/cookiecutter.json create mode 100644 templates/model_mlcube/hooks/post_gen_project.py create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json new file mode 100644 index 000000000..07cd7e295 --- /dev/null +++ b/templates/model_mlcube/cookiecutter.json @@ -0,0 +1,8 @@ +{ + "project_name": "Evaluator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "description": "Evaluator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest" +} \ No newline at end of file diff --git a/templates/model_mlcube/hooks/post_gen_project.py b/templates/model_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..2864b0326 --- /dev/null +++ b/templates/model_mlcube/hooks/post_gen_project.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +import os + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + data_path = "mlcube/workspace/data" + preds_path = "mlcube/workspace/predictions" + + paths = [preds_path, data_path] + for path in paths: + os.makedirs(path, exist_ok=True) diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..a51a8b374 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,29 @@ +name: {{ cookiecutter.project_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + infer: + # Computes predictions on input data + parameters: + inputs: { + data_path: data/, + parameters_file: parameters.yaml, + # Feel free to include other files required for inference. + # These files MUST go inside the additional_files path. + # e.g. model weights + # weights: additional_files/weights.pt, + } + outputs: {output_path: {type: directory, default: predictions}} \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py new file mode 100644 index 000000000..4c6fb1f4a --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -0,0 +1,28 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("infer") +def prepare( + data_path: str = typer.Option(..., "--data_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), + # Provide additional parameters as described in the mlcube.yaml file + # e.g. model weights: + # weights: str = typer.Option(..., "--weights"), +): + # Modify the prepare command as needed + raise NotImplementedError("The evaluate method is not yet implemented") + + +@app.command("hotfix") +def hotfix(): + # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY + pass + + +if __name__ == "__main__": + app() From 7338755717d55cca03c8607978e06cdf4e2768a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:25:27 -0500 Subject: [PATCH 16/52] Update cookiecutter variable default values --- templates/model_mlcube/cookiecutter.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json index 07cd7e295..e354d1d6d 100644 --- a/templates/model_mlcube/cookiecutter.json +++ b/templates/model_mlcube/cookiecutter.json @@ -1,7 +1,7 @@ { - "project_name": "Evaluator MLCube", + "project_name": "Model MLCube", "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Evaluator MLCube Template. Provided by MLCommons", + "description": "Model MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest" From 3ae92263b4c303158e66ad7eca8c8bf4c8bc85b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:41:10 -0500 Subject: [PATCH 17/52] Create medperf CLI command for creating MLCubes --- cli/medperf/commands/mlcube/create.py | 24 ++++++++++++++++++++++++ cli/medperf/commands/mlcube/mlcube.py | 13 +++++++++++++ cli/medperf/config.py | 7 +++++++ 3 files changed, 44 insertions(+) create mode 100644 cli/medperf/commands/mlcube/create.py diff --git a/cli/medperf/commands/mlcube/create.py b/cli/medperf/commands/mlcube/create.py new file mode 100644 index 000000000..f1222b42b --- /dev/null +++ b/cli/medperf/commands/mlcube/create.py @@ -0,0 +1,24 @@ +from cookiecutter.main import cookiecutter + +from medperf import config +from medperf.exceptions import InvalidArgumentError + + +class CreateCube: + @classmethod + def run(cls, template_name: str): + """Creates a new MLCube based on one of the provided templates + + Args: + template_name (str): The name of the template to use + """ + repo = config.github_repository + template_dirs = config.templates + if template_name not in template_dirs: + templates = list(template_dirs.keys()) + raise InvalidArgumentError( + f"Invalid template name. Available templates: [{' | '.join(templates)}]" + ) + + template_dir = template_dirs[template_name] + cookiecutter(repo, directory=template_dir) diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index 64391ebb1..a6c5a0a80 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -7,6 +7,7 @@ from medperf.entities.cube import Cube from medperf.commands.list import EntityList from medperf.commands.view import EntityView +from medperf.commands.mlcube.create import CreateCube from medperf.commands.mlcube.submit import SubmitCube from medperf.commands.mlcube.associate import AssociateCube @@ -28,6 +29,18 @@ def list( ) +@app.command("create") +@clean_except +def create( + template: str = typer.Argument( + None, + help=f"MLCube template name. Available templates: [{' | '.join(config.templates.keys())}]", + ) +): + """Creates an MLCube based on one of the specified templates""" + CreateCube.run(template) + + @app.command("submit") @clean_except def submit( diff --git a/cli/medperf/config.py b/cli/medperf/config.py index c5f081cbe..2f9e5062c 100644 --- a/cli/medperf/config.py +++ b/cli/medperf/config.py @@ -70,3 +70,10 @@ "platform", "cleanup", ] + +github_repository = "https://github.com/aristizabal95/medperf-2" +templates = { + "data_preparator": "templates/data_preparator_mlcube", + "model": "templates/model_mlcube", + "evaluator": "templates/evaluator_mlcube", +} From e07cde26c241b13c59ca545f0955f617f02a6202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 16:01:06 -0500 Subject: [PATCH 18/52] Provide additional options for mlcube create --- cli/medperf/commands/mlcube/create.py | 16 ++++++++++++++-- cli/medperf/commands/mlcube/mlcube.py | 13 +++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cli/medperf/commands/mlcube/create.py b/cli/medperf/commands/mlcube/create.py index f1222b42b..b87026ee3 100644 --- a/cli/medperf/commands/mlcube/create.py +++ b/cli/medperf/commands/mlcube/create.py @@ -6,11 +6,13 @@ class CreateCube: @classmethod - def run(cls, template_name: str): + def run(cls, template_name: str, output_path: str = ".", config_file: str = None): """Creates a new MLCube based on one of the provided templates Args: template_name (str): The name of the template to use + output_path (str, Optional): The desired path for the MLCube. Defaults to current path. + config_file (str, Optional): Path to a JSON configuration file. If not passed, user is prompted. """ repo = config.github_repository template_dirs = config.templates @@ -20,5 +22,15 @@ def run(cls, template_name: str): f"Invalid template name. Available templates: [{' | '.join(templates)}]" ) + no_input = False + if config_file is not None: + no_input = True + template_dir = template_dirs[template_name] - cookiecutter(repo, directory=template_dir) + cookiecutter( + repo, + directory=template_dir, + output_dir=output_path, + config_file=config_file, + no_input=no_input, + ) diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index a6c5a0a80..4a411287d 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -35,10 +35,19 @@ def create( template: str = typer.Argument( None, help=f"MLCube template name. Available templates: [{' | '.join(config.templates.keys())}]", - ) + ), + output_path: str = typer.Option( + ".", "--output", "-o", help="Save the generated MLCube to the specified path" + ), + config_file: str = typer.Option( + None, + "--config-file", + "-c", + help="JSON Configuration file. If not present then user is prompted for configuration", + ), ): """Creates an MLCube based on one of the specified templates""" - CreateCube.run(template) + CreateCube.run(template, output_path, config_file) @app.command("submit") From 68e136a0d8eddf85a76e1ec709d8a23763884886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 11:29:06 -0500 Subject: [PATCH 19/52] Start working on tests --- .../tests/commands/mlcube/test_create.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cli/medperf/tests/commands/mlcube/test_create.py diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py new file mode 100644 index 000000000..b9be20a69 --- /dev/null +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -0,0 +1,46 @@ +import pytest +from unittest.mock import ANY + +from medperf import config +from medperf.commands.mlcube.create import CreateCube +from medperf.exceptions import InvalidArgumentError + +PATCH_CREATE = "medperf.commands.mlcube.create.{}" + + +@pytest.fixture +def setup(mocker): + spy = mocker.patch(PATCH_CREATE.format("cookiecutter")) + return spy + + +class TestTemplate: + @pytest.mark.parametrize("template,dir", list(config.templates.items())) + def test_valid_template_is_used(mocker, setup, template, dir): + # Arrange + spy = setup + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "directory" in spy.call_args.kwargs + assert spy.call_args.kwargs["directory"] == dir + + @pytest.mark.parametrize("template", ["invalid"]) + def test_invalid_template_raises_error(mocker, template): + # Act & Assert + with pytest.raises(InvalidArgumentError): + CreateCube.run(template) + + +class TestOutputPath: + def test_current_path_is_used_by_default(): + # TODO + pass + + @pytest.mark.paramterize("output_path", ["first/path", "second/path"]) + def test_output_path_is_used_for_template_creation(): + # TODO + pass From b8e03acf0a058c395bb41a7bc578d589ccd64deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:41:48 -0500 Subject: [PATCH 20/52] Add tests for cube create --- .../tests/commands/mlcube/test_create.py | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py index b9be20a69..08ed19925 100644 --- a/cli/medperf/tests/commands/mlcube/test_create.py +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -36,11 +36,74 @@ def test_invalid_template_raises_error(mocker, template): class TestOutputPath: - def test_current_path_is_used_by_default(): - # TODO - pass - - @pytest.mark.paramterize("output_path", ["first/path", "second/path"]) - def test_output_path_is_used_for_template_creation(): - # TODO - pass + def test_current_path_is_used_by_default(mocker, setup): + # Arrange + path = "." + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "output_dir" in spy.call_args.kwargs + assert spy.call_args.kwargs["output_dir"] == path + + @pytest.mark.parametrize("output_path", ["first/path", "second/path"]) + def test_output_path_is_used_for_template_creation(mocker, setup, output_path): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, output_path=output_path) + + # Assert + spy.assert_called_once() + assert "output_dir" in spy.call_args.kwargs + assert spy.call_args.kwargs["output_dir"] == output_path + + +class TestConfigFile: + def test_config_file_is_disabled_by_default(mocker, setup): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "config_file" in spy.call_args.kwargs + assert spy.call_args.kwargs["config_file"] is None + + @pytest.mark.parametrize("config_file", ["path/to/config.json"]) + def test_config_file_is_used_when_passed(mocker, setup, config_file): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, config_file=config_file) + + # Assert + spy.assert_called_once() + assert "config_file" in spy.call_args.kwargs + assert spy.call_args.kwargs["config_file"] is config_file + + @pytest.mark.parametrize("config_file", [None, "config.json"]) + def test_passing_config_file_disables_input(mocker, setup, config_file): + # Arrange + spy = setup + should_not_input = config_file is not None + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, config_file=config_file) + + # Assert + spy.assert_called_once() + assert "no_input" in spy.call_args.kwargs + assert spy.call_args.kwargs["no_input"] == should_not_input From 7896b256d1876be6106575c6272e1cb4817f9aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:50:31 -0500 Subject: [PATCH 21/52] Ignore invalid syntax on cookiecutter conditionals --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 6 +++--- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index a617061b9..f78c4a4cd 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -20,9 +20,9 @@ tasks: inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} outputs: { output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ - {% endif %} + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 + output_labels_path: labels/ # noqa: E999 + {% endif %} # noqa: E999 } sanity_check: parameters: diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..3892ef2bc 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} + {% endif %} # noqa: E999 ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From 4f78981d3a337e452b1bbcdcfb2fbe79c34b021b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:55:41 -0500 Subject: [PATCH 22/52] Ignore more flake8 errors --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 3892ef2bc..04afcb3a3 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 E225 output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} # noqa: E999 + {% endif %} # noqa: E999 E225 ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From f5dab5ecd3edb94b1370b1b7cf489a371b640f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:57:32 -0500 Subject: [PATCH 23/52] Remove unused import --- cli/medperf/tests/commands/mlcube/test_create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py index 08ed19925..da52d72ab 100644 --- a/cli/medperf/tests/commands/mlcube/test_create.py +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -1,5 +1,4 @@ import pytest -from unittest.mock import ANY from medperf import config from medperf.commands.mlcube.create import CreateCube From a03d7f6aa16ba654312b69eef616005474ac067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 10:12:06 -0500 Subject: [PATCH 24/52] Empty commit for cloudbuild From 6bb60d0460a5f3b530bcf71afd0ee68d90a367d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:41:35 -0500 Subject: [PATCH 25/52] Fix inconsistency with labels paths --- .../data_preparator_mlcube/hooks/post_gen_project.py | 8 ++++---- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index 540b83bd7..68a54ddc9 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -6,14 +6,14 @@ if __name__ == "__main__": input_data_path = "mlcube/workspace/input_data" + input_labels_path = "mlcube/workspace/input_labels" data_path = "mlcube/workspace/data" - labels_path = "mlcube/workspace/labels" - paths = [input_data_path, data_path, labels_path] + paths = [input_data_path, input_labels_path, data_path] if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = "mlcube/workspace/input_labels" - paths.append(input_labels_path) + labels_path = "mlcube/workspace/labels" + paths.append(labels_path) for path in paths: os.makedirs(path, exist_ok=True) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index f78c4a4cd..aa4f4e4f8 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -22,6 +22,8 @@ tasks: output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 output_labels_path: labels/ # noqa: E999 + {% else %} # noqa: E999 + # output_labels_path: labels/ {% endif %} # noqa: E999 } sanity_check: From 43b6cabdd095cef591d0a2c0d866a0c6dec84b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:45:40 -0500 Subject: [PATCH 26/52] Update mlcube.yaml so it can be commented on docs --- .../mlcube/mlcube.yaml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index aa4f4e4f8..db9855b5c 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -17,7 +17,11 @@ docker: tasks: prepare: parameters: - inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + inputs: { + data_path: input_data, + labels_path: input_labels, + parameters_file: parameters.yaml + } outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 @@ -28,8 +32,16 @@ tasks: } sanity_check: parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} + inputs: { + data_path: data/, + parameters_file: parameters.yaml + } statistics: parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file + inputs: { + data_path: data/, + parameters_file: parameters.yaml + } + outputs: { + output_path: {type: file, default: statistics.yaml} + } \ No newline at end of file From 55b5d22a170b72dcb7899b50b479925080f13239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:53:51 -0500 Subject: [PATCH 27/52] Don't render noqa comments on template --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index db9855b5c..4041f9ccb 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -25,10 +25,10 @@ tasks: outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 - output_labels_path: labels/ # noqa: E999 - {% else %} # noqa: E999 + output_labels_path: labels/ {# noqa: E999} + {% else %} {# noqa: E999} # output_labels_path: labels/ - {% endif %} # noqa: E999 + {% endif %} {# noqa: E999} } sanity_check: parameters: From 135c59846e3e0736bcf6672127bc2d4a6cffd20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:55:56 -0500 Subject: [PATCH 28/52] Remove flake8 specific ignores --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 4041f9ccb..43dcbb712 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -24,11 +24,11 @@ tasks: } outputs: { output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 - output_labels_path: labels/ {# noqa: E999} - {% else %} {# noqa: E999} + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% else %} # output_labels_path: labels/ - {% endif %} {# noqa: E999} + {% endif %} } sanity_check: parameters: From e9e2c329a06a720297331ca688f6527bfab98dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:56:42 -0500 Subject: [PATCH 29/52] Exclude templates from lint checks --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 02d373692..a8d0ca017 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -33,7 +33,7 @@ jobs: # Exclude examples folder as it doesn't contain code related to medperf tools # Exclude migrations folder as it contains autogenerated code # Ignore E231, as it is raising warnings with auto-generated code. - flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*" + flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*",templates/ - name: Test with pytest run: | pytest From e95dab89ec5513e9c0a728aa02a946622ac9eb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:57:17 -0500 Subject: [PATCH 30/52] Remove specific flake8 ignores --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 04afcb3a3..7b38614c9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 E225 + {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} # noqa: E999 E225 + {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From d059b7a3ea46e80d121b65ea8f75a5cab3c7a8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 10 Mar 2023 14:27:30 -0500 Subject: [PATCH 31/52] Fix labels_paht being passed in he wrong situation --- .../mlcube/mlcube.yaml | 14 ++++++++++++-- .../project/mlcube.py | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 43dcbb712..e546012d2 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -25,21 +25,31 @@ tasks: outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ + output_labels_path: labels/, {% else %} - # output_labels_path: labels/ + # output_labels_path: labels/, {% endif %} } sanity_check: parameters: inputs: { data_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + labels_path: labels/, + {% else %} + # labels_path: labels/, + {% endif %} parameters_file: parameters.yaml } statistics: parameters: inputs: { data_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + labels_path: labels/, + {% else %} + # labels_path: labels/, + {% endif %} parameters_file: parameters.yaml } outputs: { diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..36cb01de1 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -22,7 +22,9 @@ def prepare( @app.command("sanity_check") def sanity_check( data_path: str = typer.Option(..., "--data_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} labels_path: str = typer.Option(..., "--labels_path"), + {% endif %} parameters_file: str = typer.Option(..., "--parameters_file"), ): # Modify the sanity_check command as needed @@ -32,7 +34,9 @@ def sanity_check( @app.command("statistics") def sanity_check( data_path: str = typer.Option(..., "--data_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} labels_path: str = typer.Option(..., "--labels_path"), + {% endif %} parameters_file: str = typer.Option(..., "--parameters_file"), out_path: str = typer.Option(..., "--output_path"), ): From fcdaa7bc05c68bda03ea49d7679bc89156730695 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 13 Mar 2023 10:01:28 -0500 Subject: [PATCH 32/52] Add requirements to cookiecutters --- .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file From 37f3f3ce1277295a62ec82983c4fefd1edda1bd7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Tue, 14 Mar 2023 10:26:12 -0500 Subject: [PATCH 33/52] Set separate labels as true by default --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index 95b9a12d5..1eb3db7de 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -5,5 +5,5 @@ "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest", - "use_separate_output_labels": "n" + "use_separate_output_labels": "y" } \ No newline at end of file From 7a33c23c74768b4ff4df1e156a8d05f3fee37db7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Fri, 31 Mar 2023 11:42:00 -0500 Subject: [PATCH 34/52] Remove duplicate templates --- .../data_preparator_mlcube/cookiecutter.json | 9 --- .../hooks/post_gen_project.py | 19 ------- .../mlcube/mlcube.yaml | 57 ------------------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 48 ---------------- .../project/requirements.txt | 2 - templates/evaluator_mlcube/cookiecutter.json | 8 --- .../hooks/post_gen_project.py | 13 ----- .../mlcube/mlcube.yaml | 22 ------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 26 --------- .../project/requirements.txt | 2 - templates/model_mlcube/cookiecutter.json | 8 --- .../model_mlcube/hooks/post_gen_project.py | 13 ----- .../mlcube/mlcube.yaml | 29 ---------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 28 --------- .../project/requirements.txt | 2 - 21 files changed, 379 deletions(-) delete mode 100644 templates/data_preparator_mlcube/cookiecutter.json delete mode 100644 templates/data_preparator_mlcube/hooks/post_gen_project.py delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt delete mode 100644 templates/evaluator_mlcube/cookiecutter.json delete mode 100644 templates/evaluator_mlcube/hooks/post_gen_project.py delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt delete mode 100644 templates/model_mlcube/cookiecutter.json delete mode 100644 templates/model_mlcube/hooks/post_gen_project.py delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json deleted file mode 100644 index 1eb3db7de..000000000 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "project_name": "Data Preparator MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Data Preparator MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest", - "use_separate_output_labels": "y" -} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py deleted file mode 100644 index 68a54ddc9..000000000 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - input_data_path = "mlcube/workspace/input_data" - input_labels_path = "mlcube/workspace/input_labels" - data_path = "mlcube/workspace/data" - - paths = [input_data_path, input_labels_path, data_path] - - if "{{ cookiecutter.use_separate_output_labels }}" == "y": - labels_path = "mlcube/workspace/labels" - paths.append(labels_path) - - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index e546012d2..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - prepare: - parameters: - inputs: { - data_path: input_data, - labels_path: input_labels, - parameters_file: parameters.yaml - } - outputs: { - output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/, - {% else %} - # output_labels_path: labels/, - {% endif %} - } - sanity_check: - parameters: - inputs: { - data_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: labels/, - {% else %} - # labels_path: labels/, - {% endif %} - parameters_file: parameters.yaml - } - statistics: - parameters: - inputs: { - data_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: labels/, - {% else %} - # labels_path: labels/, - {% endif %} - parameters_file: parameters.yaml - } - outputs: { - output_path: {type: file, default: statistics.yaml} - } \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 36cb01de1..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,48 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("prepare") -def prepare( - data_path: str = typer.Option(..., "--data_path"), - labels_path: str = typer.Option(..., "--labels_path"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} -): - # Modify the prepare command as needed - raise NotImplementedError("The prepare method is not yet implemented") - - -@app.command("sanity_check") -def sanity_check( - data_path: str = typer.Option(..., "--data_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: str = typer.Option(..., "--labels_path"), - {% endif %} - parameters_file: str = typer.Option(..., "--parameters_file"), -): - # Modify the sanity_check command as needed - raise NotImplementedError("The sanity check method is not yet implemented") - - -@app.command("statistics") -def sanity_check( - data_path: str = typer.Option(..., "--data_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: str = typer.Option(..., "--labels_path"), - {% endif %} - parameters_file: str = typer.Option(..., "--parameters_file"), - out_path: str = typer.Option(..., "--output_path"), -): - # Modify the statistics command as needed - raise NotImplementedError("The statistics method is not yet implemented") - - -if __name__ == "__main__": - app() diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json deleted file mode 100644 index 07cd7e295..000000000 --- a/templates/evaluator_mlcube/cookiecutter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "project_name": "Evaluator MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Evaluator MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest" -} \ No newline at end of file diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py deleted file mode 100644 index e36bdaef0..000000000 --- a/templates/evaluator_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - preds_path = "mlcube/workspace/predictions" - labels_path = "mlcube/workspace/labels" - - paths = [preds_path, labels_path] - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index c486308c9..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - evaluate: - # Computes evaluation metrics on the given predictions and ground truths - parameters: - inputs: {predictions: predictions, labels: labels, parameters_file: parameters.yaml} - outputs: {output_path: {type: "file", default: "results.yaml"}} \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 1de721eb9..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,26 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("evaluate") -def prepare( - labels: str = typer.Option(..., "--labels"), - predictions: str = typer.Option(..., "--predictions"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), -): - # Modify the prepare command as needed - raise NotImplementedError("The evaluate method is not yet implemented") - - -@app.command("hotfix") -def hotfix(): - # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY - pass - - -if __name__ == "__main__": - app() diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json deleted file mode 100644 index e354d1d6d..000000000 --- a/templates/model_mlcube/cookiecutter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "project_name": "Model MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Model MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest" -} \ No newline at end of file diff --git a/templates/model_mlcube/hooks/post_gen_project.py b/templates/model_mlcube/hooks/post_gen_project.py deleted file mode 100644 index 2864b0326..000000000 --- a/templates/model_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - data_path = "mlcube/workspace/data" - preds_path = "mlcube/workspace/predictions" - - paths = [preds_path, data_path] - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index a51a8b374..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - infer: - # Computes predictions on input data - parameters: - inputs: { - data_path: data/, - parameters_file: parameters.yaml, - # Feel free to include other files required for inference. - # These files MUST go inside the additional_files path. - # e.g. model weights - # weights: additional_files/weights.pt, - } - outputs: {output_path: {type: directory, default: predictions}} \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 4c6fb1f4a..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,28 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("infer") -def prepare( - data_path: str = typer.Option(..., "--data_path"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), - # Provide additional parameters as described in the mlcube.yaml file - # e.g. model weights: - # weights: str = typer.Option(..., "--weights"), -): - # Modify the prepare command as needed - raise NotImplementedError("The evaluate method is not yet implemented") - - -@app.command("hotfix") -def hotfix(): - # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY - pass - - -if __name__ == "__main__": - app() diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file From 1a79ae827f4da609707c875e8a8949f689650798 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Fri, 28 Apr 2023 11:44:46 -0500 Subject: [PATCH 35/52] Implement update method for bmk, mlcube --- cli/medperf/entities/benchmark.py | 10 +++++ cli/medperf/entities/cube.py | 69 +++++++++++++++++++++++++++---- cli/medperf/entities/interface.py | 8 ++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index 1899b259a..7f73f1ef8 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -267,6 +267,16 @@ def get_models_uids(cls, benchmark_uid: int) -> List[int]: """ return config.comms.get_benchmark_models(benchmark_uid) + def update(self, **kwargs): + """Updates a benchmark with the given property-value pairs + """ + data = self.todict() + data.update(kwargs) + # Edit entity by creating a new one. This ensures any extra steps + # that depend on the fields are being taken care of + new_bmk = Benchmark(**data) + self.__dict__ = new_bmk.__dict__ + def todict(self) -> dict: """Dictionary representation of the benchmark instance diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index bbd787dd3..edfaaa25f 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -63,6 +63,39 @@ def __init__(self, *args, **kwargs): if self.git_parameters_url: self.params_path = os.path.join(path, config.params_filename) + @classmethod + def __is_valid_update(cls, old_cube: "Cube", new_cube: "Cube") -> bool: + """Determines if an update is valid given the changes between entities + + Args: + old_cube (Cube): The old version of the MLCube + new_cube (Cube): The new version of the same MLCube + + Returns: + bool: Wether updating the old_cube to the new_cube is possible + """ + if old_cube.id != new_cube.id: + # id change should not be possible in any instance + return False + + if old_cube.state == "DEVELOPMENT": + # Development entities can be freely edited + return True + + production_inmutable_fields = { + "mlcube_hash", + "parameters_hash", + "image_tarball_hash", + "additional_files_tarball_hash" + } + + updated_field_values = set(new_cube.items()) - set(old_cube.items()) + updated_fields = {field for field, val in updated_field_values} + updated_inmutable_fields = updated_fields.intersection(production_inmutable_fields) + + if len(updated_inmutable_fields): + return False + @classmethod def all(cls, local_only: bool = False, filters: dict = {}) -> List["Cube"]: """Class method for retrieving all retrievable MLCubes @@ -234,15 +267,22 @@ def download_image(self): proc.close() return "" - def download(self): + def download(self, provided_fields: List[str] = None): """Downloads the required elements for an mlcube to run locally.""" + try: + local_hashes = self.get_local_hashes() + except FileNotFoundError: + local_hashes = {} + + if provided_fields is None or "git_mlcube_url" in provided_fields: + local_hashes["mlcube_hash"] = self.download_mlcube() + if provided_fields is None or "git_parameters_url" in provided_fields: + local_hashes["parameters_hash"] = self.download_parameters() + if provided_fields is None or "additional_files_tarball_url" in provided_fields: + local_hashes["additional_files_tarball_hash"] = self.download_additional() + if provided_fields is None or "image_tarball_url" in provided_fields: + local_hashes["image_tarball_hash"] = self.download_image() - local_hashes = { - "mlcube_hash": self.download_mlcube(), - "parameters_hash": self.download_parameters(), - "additional_files_tarball_hash": self.download_additional(), - "image_tarball_hash": self.download_image(), - } self.store_local_hashes(local_hashes) def valid(self) -> bool: @@ -335,6 +375,21 @@ def get_default_output(self, task: str, out_key: str, param_key: str = None) -> return out_path + def update(self, **kwargs): + """Updates a cube with the given property-value pairs + """ + data = self.todict() + data.update(kwargs) + new_cube = Cube(**data) + + provided_fields = set(kwargs.keys()) + + # Download any files that were modified + self.download(provided_fields) + + if new_cube.is_valid() and Cube.__is_valid_update(self, new_cube): + self.__dict__ = new_cube.__dict__ + def todict(self) -> Dict: return self.extended_dict() diff --git a/cli/medperf/entities/interface.py b/cli/medperf/entities/interface.py index af2afabd7..89e9ecc91 100644 --- a/cli/medperf/entities/interface.py +++ b/cli/medperf/entities/interface.py @@ -40,6 +40,14 @@ def todict(self) -> Dict: Dict: Dictionary containing information about the entity """ + @abstractmethod + def update(self, **kwargs): + """Updates the current entity with the given fields + + Arguments: + kwargs (dict): Key-value pair of properties to edit and their corresponding new values + """ + @abstractmethod def write(self) -> str: """Writes the entity to the local storage From 78155973b4865302e598c55532c6a67d116b5a21 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Thu, 4 May 2023 17:03:19 -0500 Subject: [PATCH 36/52] Implement edit/update methods. Add bmk dset cmd --- cli/medperf/commands/benchmark/benchmark.py | 84 +++++++++++++++------ cli/medperf/commands/dataset/dataset.py | 37 +++++++-- cli/medperf/commands/edit.py | 35 +++++++++ cli/medperf/entities/benchmark.py | 58 ++++++++++++-- cli/medperf/entities/cube.py | 61 +++++++++------ cli/medperf/entities/dataset.py | 65 +++++++++++++++- cli/medperf/entities/interface.py | 25 ++++-- 7 files changed, 297 insertions(+), 68 deletions(-) create mode 100644 cli/medperf/commands/edit.py diff --git a/cli/medperf/commands/benchmark/benchmark.py b/cli/medperf/commands/benchmark/benchmark.py index 2b32608fd..b72c6190a 100644 --- a/cli/medperf/commands/benchmark/benchmark.py +++ b/cli/medperf/commands/benchmark/benchmark.py @@ -7,10 +7,35 @@ from medperf.entities.benchmark import Benchmark from medperf.commands.list import EntityList from medperf.commands.view import EntityView +from medperf.commands.edit import EntityEdit from medperf.commands.benchmark.submit import SubmitBenchmark from medperf.commands.benchmark.associate import AssociateBenchmark from medperf.commands.result.create import BenchmarkExecution +NAME_OPTION = typer.Option(..., "--name", "-n", help="Name of the benchmark") +DESC_OPTION = typer.Option( + ..., "--description", "-d", help="Description of the benchmark" +) +DOCS_OPTION = typer.Option("", "--docs-url", "-u", help="URL to documentation") +DEMO_URL_OPTION = typer.Option( + "", + "--demo-url", + help="""Identifier to download the demonstration dataset tarball file.\n + See `medperf mlcube submit --help` for more information""" +) +DEMO_HASH_OPTION = typer.Option( + "", "--demo-hash", help="SHA1 of demonstration dataset tarball file" +) +DATA_PREP_OPTION = typer.Option( + ..., "--data-preparation-mlcube", "-p", help="Data Preparation MLCube UID" +) +MODEL_OPTION = typer.Option( + ..., "--reference-model-mlcube", "-m", help="Reference Model MLCube UID" +) +EVAL_OPTION = typer.Option( + ..., "--evaluator-mlcube", "-e", help="Evaluator MLCube UID" +) + app = typer.Typer() @@ -32,29 +57,14 @@ def list( @app.command("submit") @clean_except def submit( - name: str = typer.Option(..., "--name", "-n", help="Name of the benchmark"), - description: str = typer.Option( - ..., "--description", "-d", help="Description of the benchmark" - ), - docs_url: str = typer.Option("", "--docs-url", "-u", help="URL to documentation"), - demo_url: str = typer.Option( - "", - "--demo-url", - help="""Identifier to download the demonstration dataset tarball file.\n - See `medperf mlcube submit --help` for more information""", - ), - demo_hash: str = typer.Option( - "", "--demo-hash", help="SHA1 of demonstration dataset tarball file" - ), - data_preparation_mlcube: int = typer.Option( - ..., "--data-preparation-mlcube", "-p", help="Data Preparation MLCube UID" - ), - reference_model_mlcube: int = typer.Option( - ..., "--reference-model-mlcube", "-m", help="Reference Model MLCube UID" - ), - evaluator_mlcube: int = typer.Option( - ..., "--evaluator-mlcube", "-e", help="Evaluator MLCube UID" - ), + name: str = NAME_OPTION, + description: str = DESC_OPTION, + docs_url: str = DOCS_OPTION, + demo_url: str = DEMO_URL_OPTION, + demo_hash: str = DEMO_HASH_OPTION, + data_preparation_mlcube: int = DATA_PREP_OPTION, + reference_model_mlcube: int = MODEL_OPTION, + evaluator_mlcube: int = EVAL_OPTION ): """Submits a new benchmark to the platform""" benchmark_info = { @@ -72,6 +82,34 @@ def submit( config.ui.print("✅ Done!") +@app.command("edit") +@clean_except +def edit( + entity_id: int = typer.Argument(..., help="Benchmark ID"), + name: str = NAME_OPTION, + description: str = DESC_OPTION, + docs_url: str = DOCS_OPTION, + demo_url: str = DEMO_URL_OPTION, + demo_hash: str = DEMO_HASH_OPTION, + data_preparation_mlcube: int = DATA_PREP_OPTION, + reference_model_mlcube: int = MODEL_OPTION, + evaluator_mlcube: int = EVAL_OPTION +): + """Edits a benchmark""" + benchmark_info = { + "name": name, + "description": description, + "docs_url": docs_url, + "demo_dataset_tarball_url": demo_url, + "demo_dataset_tarball_hash": demo_hash, + "data_preparation_mlcube": data_preparation_mlcube, + "reference_model_mlcube": reference_model_mlcube, + "data_evaluator_mlcube": evaluator_mlcube, + } + EntityEdit.run(Benchmark, entity_id, benchmark_info) + config.ui.print("✅ Done!") + + @app.command("associate") @clean_except def associate( diff --git a/cli/medperf/commands/dataset/dataset.py b/cli/medperf/commands/dataset/dataset.py index 07c97153c..69dca0c5c 100644 --- a/cli/medperf/commands/dataset/dataset.py +++ b/cli/medperf/commands/dataset/dataset.py @@ -6,10 +6,19 @@ from medperf.entities.dataset import Dataset from medperf.commands.list import EntityList from medperf.commands.view import EntityView +from medperf.commands.edit import EntityEdit from medperf.commands.dataset.create import DataPreparation from medperf.commands.dataset.submit import DatasetRegistration from medperf.commands.dataset.associate import AssociateDataset +NAME_OPTION = typer.Option(..., "--name", help="Name of the dataset") +DESC_OPTION = typer.Option( + ..., "--description", help="Description of the dataset" +) +LOC_OPTION = typer.Option( + ..., "--location", help="Location or Institution the data belongs to" +) + app = typer.Typer() @@ -43,13 +52,9 @@ def create( labels_path: str = typer.Option( ..., "--labels_path", "-l", help="Labels file location" ), - name: str = typer.Option(..., "--name", help="Name of the dataset"), - description: str = typer.Option( - ..., "--description", help="Description of the dataset" - ), - location: str = typer.Option( - ..., "--location", help="Location or Institution the data belongs to" - ), + name: str = NAME_OPTION, + description: str = DESC_OPTION, + location: str = LOC_OPTION ): """Runs the Data preparation step for a specified benchmark and raw dataset """ @@ -87,6 +92,24 @@ def register( ) +@app.command("edit") +@clean_except +def edit( + entity_id: int = typer.Argument(..., help="Dataset ID"), + name: str = NAME_OPTION, + description: str = DESC_OPTION, + location: str = LOC_OPTION +): + """Edits a Dataset""" + dset_info = { + "name": name, + "description": description, + "location": location, + } + EntityEdit.run(Dataset, entity_id, dset_info) + config.ui.print("✅ Done!") + + @app.command("associate") @clean_except def associate( diff --git a/cli/medperf/commands/edit.py b/cli/medperf/commands/edit.py new file mode 100644 index 000000000..91c2156c8 --- /dev/null +++ b/cli/medperf/commands/edit.py @@ -0,0 +1,35 @@ +from medperf.entities.interface import Editable, Updatable +from medperf.exceptions import InvalidEntityError + +class EntityEdit: + @staticmethod + def run(entity_class, id: str, fields: dict): + """Edits and updates an entity both locally and on the server if possible + + Args: + entity (Editable): Entity to modify + fields (dict): Dicitonary of fields and values to modify + """ + editor = EntityEdit(entity_class, id, fields) + editor.validate() + editor.edit() + + def __init__(self, entity_class, id, fields): + self.entity_class = entity_class + self.id = id + self.fields = fields + + def prepare(self): + self.entity = self.entity_class(self.id) + + def validate(self): + if not isinstance(self.entity, Editable): + raise InvalidEntityError("The passed entity can't be edited") + + def edit(self): + entity = self.entity + entity.edit(self.fields) + entity.write() + + if isinstance(entity, Updatable) and entity.is_registered: + entity.update() \ No newline at end of file diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index 7f73f1ef8..1e6dc06ba 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -6,13 +6,13 @@ from pydantic import HttpUrl, Field, validator import medperf.config as config -from medperf.entities.interface import Entity, Uploadable +from medperf.entities.interface import Entity, Updatable from medperf.utils import storage_path from medperf.exceptions import CommunicationRetrievalError, InvalidArgumentError from medperf.entities.schemas import MedperfSchema, ApprovableSchema, DeployableSchema -class Benchmark(Entity, Uploadable, MedperfSchema, ApprovableSchema, DeployableSchema): +class Benchmark(Entity, Updatable, MedperfSchema, ApprovableSchema, DeployableSchema): """ Class representing a Benchmark @@ -59,6 +59,43 @@ def __init__(self, *args, **kwargs): path = os.path.join(path, self.generated_uid) self.path = path + def __validate_edit(cls, old_bmk: "Benchmark", new_bmk: "Benchmark"): + """Validates that an update is valid given the changes made + + Args: + old_bmk (Benchmark): The old version of the Benchmark + new_bmk (Benchmark): The new version of the same Benchmark + + Raises: + InvalidArgumentError: The changed fields are not mutable + """ + # Field that shouldn't ber modified directly by the user + inmutable_fields = {"id",} + + # Fields that can no longer be modified while in production + production_inmutable_fields = { + "name", + "demo_dataset_tarball_hash", + "demo_dataset_generated_uid", + "data_preparation_mlcube", + "reference_model_mlcube", + "data_evaluator_mlcube" + } + + if old_bmk.state == "PRODCUTION": + inmutable_fields = inmutable_fields.join(production_inmutable_fields) + + updated_field_values = set(new_bmk.items()) - set(old_bmk.items()) + updated_fields = {field for field, _ in updated_field_values} + updated_inmutable_fields = updated_fields.intersection(production_inmutable_fields) + + if len(updated_inmutable_fields): + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: "\ + + fields_msg \ + + ". For these changes, a new Dataset is required") + raise InvalidArgumentError(msg) + @classmethod def all(cls, local_only: bool = False, filters: dict = {}) -> List["Benchmark"]: """Gets and creates instances of all retrievable benchmarks @@ -267,16 +304,25 @@ def get_models_uids(cls, benchmark_uid: int) -> List[int]: """ return config.comms.get_benchmark_models(benchmark_uid) - def update(self, **kwargs): - """Updates a benchmark with the given property-value pairs + def edit(self, **kwargs): + """Edits a benchmark with the given property-value pairs """ data = self.todict() data.update(kwargs) - # Edit entity by creating a new one. This ensures any extra steps - # that depend on the fields are being taken care of new_bmk = Benchmark(**data) + + Benchmark.__validate_edit(self, new_bmk) + self.__dict__ = new_bmk.__dict__ + def update(self): + """Updates the benchmark on the server + """ + if not self.is_registered: + raise MedperfException("Can't update an unregistered benchmark") + body = self.todict() + config.comms.update_benchmark(body) + def todict(self) -> dict: """Dictionary representation of the benchmark instance diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index edfaaa25f..f02aab540 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -7,7 +7,7 @@ from pathlib import Path from medperf.utils import untar, combine_proc_sp_text, list_files, storage_path, cleanup -from medperf.entities.interface import Entity, Uploadable +from medperf.entities.interface import Entity, Updatable from medperf.entities.schemas import MedperfSchema, DeployableSchema from medperf.exceptions import ( InvalidArgumentError, @@ -20,7 +20,7 @@ from medperf.comms.entity_resources import resources -class Cube(Entity, Uploadable, MedperfSchema, DeployableSchema): +class Cube(Entity, Updatable, MedperfSchema, DeployableSchema): """ Class representing an MLCube Container @@ -64,24 +64,21 @@ def __init__(self, *args, **kwargs): self.params_path = os.path.join(path, config.params_filename) @classmethod - def __is_valid_update(cls, old_cube: "Cube", new_cube: "Cube") -> bool: - """Determines if an update is valid given the changes between entities + def __validate_edit(cls, old_cube: "Cube", new_cube: "Cube"): + """Validates that an edit is valid given the changes made Args: old_cube (Cube): The old version of the MLCube new_cube (Cube): The new version of the same MLCube - Returns: - bool: Wether updating the old_cube to the new_cube is possible + Raises: + InvalidEntityError: The changes created an invalid entity configuration + InvalidArugmentError: The changed fields are not mutable """ - if old_cube.id != new_cube.id: - # id change should not be possible in any instance - return False - - if old_cube.state == "DEVELOPMENT": - # Development entities can be freely edited - return True + # Fields that shouldn't be modified directly by the user + inmutable_fields = {"id",} + # Fields that can no longer be modified while in production production_inmutable_fields = { "mlcube_hash", "parameters_hash", @@ -89,12 +86,28 @@ def __is_valid_update(cls, old_cube: "Cube", new_cube: "Cube") -> bool: "additional_files_tarball_hash" } + if old_cube.state == "PRODUCTION": + inmutable_fields = inmutable_fields.join(production_inmutable_fields) + updated_field_values = set(new_cube.items()) - set(old_cube.items()) - updated_fields = {field for field, val in updated_field_values} + updated_fields = {field for field, _ in updated_field_values} + + # Download any new files + new_cube.download(updated_fields) + updated_inmutable_fields = updated_fields.intersection(production_inmutable_fields) + if not new_cube.valid(): + msg = "Invalid MLCube configuration" + raise InvalidEntityError(msg) + + if len(updated_inmutable_fields): - return False + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: "\ + + fields_msg \ + + ". For these changes, a new MLCube is required") + raise InvalidArgumentError(msg) @classmethod def all(cls, local_only: bool = False, filters: dict = {}) -> List["Cube"]: @@ -375,20 +388,24 @@ def get_default_output(self, task: str, out_key: str, param_key: str = None) -> return out_path - def update(self, **kwargs): - """Updates a cube with the given property-value pairs + def edit(self, **kwargs): + """Edits a cube with the given property-value pairs """ data = self.todict() data.update(kwargs) new_cube = Cube(**data) - provided_fields = set(kwargs.keys()) + Cube.__validate_edit(self, new_cube) - # Download any files that were modified - self.download(provided_fields) + self.__dict__ = new_cube.__dict__ - if new_cube.is_valid() and Cube.__is_valid_update(self, new_cube): - self.__dict__ = new_cube.__dict__ + def update(self): + """Updates the benchmark on the server + """ + if not self.is_registered: + raise MedperfException("Can't update an unregistered cube") + body = self.todict() + config.comms.update_mlcube(body) def todict(self) -> Dict: return self.extended_dict() diff --git a/cli/medperf/entities/dataset.py b/cli/medperf/entities/dataset.py index 3a5cc7660..04cc77efc 100644 --- a/cli/medperf/entities/dataset.py +++ b/cli/medperf/entities/dataset.py @@ -6,7 +6,7 @@ from medperf.utils import storage_path from medperf.enums import Status -from medperf.entities.interface import Entity, Uploadable +from medperf.entities.interface import Entity, Updatable from medperf.entities.schemas import MedperfSchema, DeployableSchema from medperf.exceptions import ( InvalidArgumentError, @@ -16,7 +16,7 @@ import medperf.config as config -class Dataset(Entity, Uploadable, MedperfSchema, DeployableSchema): +class Dataset(Entity, Updatable, MedperfSchema, DeployableSchema): """ Class representing a Dataset @@ -69,6 +69,48 @@ def __init__(self, *args, **kwargs): if self.separate_labels: self.labels_path = os.path.join(self.path, "labels") + @classmethod + def __validate_edit(cls, old_dset: "Dataset", new_dset: "Dataset"): + """Determines if an update is valid given the changes made + + Args: + old_dset (Dataset): The old version of the dataset + new_dset (Dataset): The updated version of the same dataset + + Raises: + InvalidArugmentError: The changed fields are not mutable + """ + + # Fields that shouldn't be modified directly by the user + inmutable_fields = { + "id", + "input_data_hash", + "generated_uid", + "separate_labels", + "generated_metadata", + "data_preparation_mlcube", + } + + # Fields that can no longer be modified while in production + production_inmutable_fields = { + "name", + "split_seed" + } + + if old_dset.state == "PRODUCTION": + inmutable_fields = inmutable_fields.join(production_inmutable_fields) + + updated_field_values = set(new_dset.items()) - set(old_dset.items()) + updated_fields = {field for field, _ in updated_field_values} + updated_inmutable_fields = updated_fields.intersection(inmutable_fields) + + if len(updated_inmutable_fields): + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: " \ + + fields_msg \ + + ". For these changes, a new dataset is required") + raise InvalidArgumentError(msg) + def todict(self): return self.extended_dict() @@ -217,6 +259,25 @@ def upload(self): updated_dataset_dict["separate_labels"] = dataset_dict["separate_labels"] return updated_dataset_dict + def edit(self, **kwargs): + """Edits a dataset with the given property-value pairs + """ + data = self.todict() + data.update(kwargs) + new_dset = Dataset(**data) + + Dataset.__validate_edit(self, new_dset) + + self.__dict__ = new_dset.__dict__ + + def update(self): + """Updates the benchmark on the server + """ + if not self.is_registered: + raise MedperfException("Can't update an unregistered dataset") + body = self.todict() + config.comms.update_dataset(body) + @classmethod def __get_local_dict(cls, data_uid): dataset_path = os.path.join(storage_path(config.data_storage), str(data_uid)) diff --git a/cli/medperf/entities/interface.py b/cli/medperf/entities/interface.py index 89e9ecc91..2126b5157 100644 --- a/cli/medperf/entities/interface.py +++ b/cli/medperf/entities/interface.py @@ -40,14 +40,6 @@ def todict(self) -> Dict: Dict: Dictionary containing information about the entity """ - @abstractmethod - def update(self, **kwargs): - """Updates the current entity with the given fields - - Arguments: - kwargs (dict): Key-value pair of properties to edit and their corresponding new values - """ - @abstractmethod def write(self) -> str: """Writes the entity to the local storage @@ -83,3 +75,20 @@ def identifier(self): @property def is_registered(self): return self.id is not None + + +class Editable: + @abstractmethod + def edit(self, **kwargs): + """Edits the current entity with the given fields + + Arguments: + kwargs (dict): Key-value pair of properties to edit and their corresponding new values + """ + + +class Updatable(Uploadable, Editable): + @abstractmethod + def update(self): + """Updates the current entity on the server + """ \ No newline at end of file From 645cbad88635825210a4efa10cdf2a5fb42389b0 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 8 May 2023 11:43:51 -0500 Subject: [PATCH 37/52] Remove editable --- cli/medperf/commands/benchmark/benchmark.py | 1 + cli/medperf/commands/edit.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/medperf/commands/benchmark/benchmark.py b/cli/medperf/commands/benchmark/benchmark.py index b72c6190a..56a1dc2ec 100644 --- a/cli/medperf/commands/benchmark/benchmark.py +++ b/cli/medperf/commands/benchmark/benchmark.py @@ -107,6 +107,7 @@ def edit( "data_evaluator_mlcube": evaluator_mlcube, } EntityEdit.run(Benchmark, entity_id, benchmark_info) + cleanup() config.ui.print("✅ Done!") diff --git a/cli/medperf/commands/edit.py b/cli/medperf/commands/edit.py index 91c2156c8..c50456ad8 100644 --- a/cli/medperf/commands/edit.py +++ b/cli/medperf/commands/edit.py @@ -1,4 +1,4 @@ -from medperf.entities.interface import Editable, Updatable +from medperf.entities.interface import Updatable from medperf.exceptions import InvalidEntityError class EntityEdit: @@ -11,6 +11,7 @@ def run(entity_class, id: str, fields: dict): fields (dict): Dicitonary of fields and values to modify """ editor = EntityEdit(entity_class, id, fields) + editor.prepare() editor.validate() editor.edit() @@ -23,7 +24,7 @@ def prepare(self): self.entity = self.entity_class(self.id) def validate(self): - if not isinstance(self.entity, Editable): + if not isinstance(self.entity, Updatable): raise InvalidEntityError("The passed entity can't be edited") def edit(self): From 70617573e6df1eff69ab8a74d0dbbf3c892b965a Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 8 May 2023 11:43:56 -0500 Subject: [PATCH 38/52] Remove editable --- cli/medperf/entities/interface.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/medperf/entities/interface.py b/cli/medperf/entities/interface.py index 2126b5157..5893f5a1e 100644 --- a/cli/medperf/entities/interface.py +++ b/cli/medperf/entities/interface.py @@ -77,7 +77,7 @@ def is_registered(self): return self.id is not None -class Editable: +class Updatable(Uploadable): @abstractmethod def edit(self, **kwargs): """Edits the current entity with the given fields @@ -86,8 +86,6 @@ def edit(self, **kwargs): kwargs (dict): Key-value pair of properties to edit and their corresponding new values """ - -class Updatable(Uploadable, Editable): @abstractmethod def update(self): """Updates the current entity on the server From 162fc56626ea774fc3ec4c080a397110e55322c0 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 10 May 2023 11:51:30 -0500 Subject: [PATCH 39/52] Use deepdiff to obtain changes between objects --- cli/medperf/entities/benchmark.py | 81 ++++++++++++++++--------------- cli/requirements.txt | 1 + 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index 1e6dc06ba..b9b2b8c09 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -2,6 +2,7 @@ from medperf.exceptions import MedperfException import yaml import logging +from deepdiff import DeepDiff from typing import List, Optional, Union from pydantic import HttpUrl, Field, validator @@ -59,43 +60,6 @@ def __init__(self, *args, **kwargs): path = os.path.join(path, self.generated_uid) self.path = path - def __validate_edit(cls, old_bmk: "Benchmark", new_bmk: "Benchmark"): - """Validates that an update is valid given the changes made - - Args: - old_bmk (Benchmark): The old version of the Benchmark - new_bmk (Benchmark): The new version of the same Benchmark - - Raises: - InvalidArgumentError: The changed fields are not mutable - """ - # Field that shouldn't ber modified directly by the user - inmutable_fields = {"id",} - - # Fields that can no longer be modified while in production - production_inmutable_fields = { - "name", - "demo_dataset_tarball_hash", - "demo_dataset_generated_uid", - "data_preparation_mlcube", - "reference_model_mlcube", - "data_evaluator_mlcube" - } - - if old_bmk.state == "PRODCUTION": - inmutable_fields = inmutable_fields.join(production_inmutable_fields) - - updated_field_values = set(new_bmk.items()) - set(old_bmk.items()) - updated_fields = {field for field, _ in updated_field_values} - updated_inmutable_fields = updated_fields.intersection(production_inmutable_fields) - - if len(updated_inmutable_fields): - fields_msg = ", ".join(updated_inmutable_fields) - msg = (f"The following fields can't be directly edited: "\ - + fields_msg \ - + ". For these changes, a new Dataset is required") - raise InvalidArgumentError(msg) - @classmethod def all(cls, local_only: bool = False, filters: dict = {}) -> List["Benchmark"]: """Gets and creates instances of all retrievable benchmarks @@ -311,9 +275,48 @@ def edit(self, **kwargs): data.update(kwargs) new_bmk = Benchmark(**data) - Benchmark.__validate_edit(self, new_bmk) + self.__validate_edit(new_bmk) + + self.__dict__.update(**new_bmk.__dict__) - self.__dict__ = new_bmk.__dict__ + def __validate_edit(self, new_bmk: "Benchmark"): + """Validates that an update is valid given the changes made + + Args: + old_bmk (Benchmark): The old version of the Benchmark + new_bmk (Benchmark): The new version of the same Benchmark + + Raises: + InvalidArgumentError: The changed fields are not mutable + """ + old_bmk = self + # Field that shouldn't ber modified directly by the user + inmutable_fields = {"id",} + + # Fields that can no longer be modified while in production + production_inmutable_fields = { + "name", + "demo_dataset_tarball_hash", + "demo_dataset_generated_uid", + "data_preparation_mlcube", + "reference_model_mlcube", + "data_evaluator_mlcube" + } + + if old_bmk.state == "PRODCUTION": + inmutable_fields = inmutable_fields.join(production_inmutable_fields) + + bmk_diffs = DeepDiff(new_bmk.todict(), old_bmk.todict()) + updated_fields = set(bmk_diffs.affected_root_keys) + + updated_inmutable_fields = updated_fields.intersection(inmutable_fields) + + if len(updated_inmutable_fields): + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: "\ + + fields_msg \ + + ". For these changes, a new Dataset is required") + raise InvalidArgumentError(msg) def update(self): """Updates the benchmark on the server diff --git a/cli/requirements.txt b/cli/requirements.txt index 4b5f11746..c55238e46 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -16,3 +16,4 @@ mlcube-singularity==0.0.9 validators==0.18.2 merge-args==0.1.4 synapseclient==2.7.0 +deepdiff==6.3.0 From 11fdf81a6036a891b8eec5ad02bb0aaf678ce7c4 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 10 May 2023 11:52:53 -0500 Subject: [PATCH 40/52] Reuse field help message --- cli/medperf/commands/benchmark/benchmark.py | 74 ++++++++++----------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/cli/medperf/commands/benchmark/benchmark.py b/cli/medperf/commands/benchmark/benchmark.py index 56a1dc2ec..1bdd5dde9 100644 --- a/cli/medperf/commands/benchmark/benchmark.py +++ b/cli/medperf/commands/benchmark/benchmark.py @@ -12,29 +12,15 @@ from medperf.commands.benchmark.associate import AssociateBenchmark from medperf.commands.result.create import BenchmarkExecution -NAME_OPTION = typer.Option(..., "--name", "-n", help="Name of the benchmark") -DESC_OPTION = typer.Option( - ..., "--description", "-d", help="Description of the benchmark" -) -DOCS_OPTION = typer.Option("", "--docs-url", "-u", help="URL to documentation") -DEMO_URL_OPTION = typer.Option( - "", - "--demo-url", - help="""Identifier to download the demonstration dataset tarball file.\n +NAME_HELP = "Name of the benchmark" +DESC_HELP = "Description of the benchmark" +DOCS_HELP = "URL to documentation" +DEMO_URL_HELP = """Identifier to download the demonstration dataset tarball file.\n See `medperf mlcube submit --help` for more information""" -) -DEMO_HASH_OPTION = typer.Option( - "", "--demo-hash", help="SHA1 of demonstration dataset tarball file" -) -DATA_PREP_OPTION = typer.Option( - ..., "--data-preparation-mlcube", "-p", help="Data Preparation MLCube UID" -) -MODEL_OPTION = typer.Option( - ..., "--reference-model-mlcube", "-m", help="Reference Model MLCube UID" -) -EVAL_OPTION = typer.Option( - ..., "--evaluator-mlcube", "-e", help="Evaluator MLCube UID" -) +DEMO_HASH_HELP = "SHA1 of demonstration dataset tarball file" +DATA_PREP_HELP = "Data Preparation MLCube UID" +MODEL_HELP = "Reference Model MLCube UID" +EVAL_HELP = "Evaluator MLCube UID" app = typer.Typer() @@ -57,14 +43,20 @@ def list( @app.command("submit") @clean_except def submit( - name: str = NAME_OPTION, - description: str = DESC_OPTION, - docs_url: str = DOCS_OPTION, - demo_url: str = DEMO_URL_OPTION, - demo_hash: str = DEMO_HASH_OPTION, - data_preparation_mlcube: int = DATA_PREP_OPTION, - reference_model_mlcube: int = MODEL_OPTION, - evaluator_mlcube: int = EVAL_OPTION + name: str = typer.Option(..., "--name", "-n", help=NAME_HELP), + description: str = typer.Option(..., "--description", "-d", help=DESC_HELP), + docs_url: str = typer.Option("", "--docs-url", "-u", help=DOCS_HELP), + demo_url: str = typer.Option("","--demo-url",help=DEMO_URL_HELP), + demo_hash: str = typer.Option("", "--demo-hash", help=DEMO_HASH_HELP), + data_preparation_mlcube: int = typer.Option( + ..., "--data-preparation-mlcube", "-p", help=DATA_PREP_HELP + ), + reference_model_mlcube: int = typer.Option( + ..., "--reference-model-mlcube", "-m", help=MODEL_HELP + ), + evaluator_mlcube: int = typer.Option( + ..., "--evaluator-mlcube", "-e", help=EVAL_HELP + ) ): """Submits a new benchmark to the platform""" benchmark_info = { @@ -86,14 +78,20 @@ def submit( @clean_except def edit( entity_id: int = typer.Argument(..., help="Benchmark ID"), - name: str = NAME_OPTION, - description: str = DESC_OPTION, - docs_url: str = DOCS_OPTION, - demo_url: str = DEMO_URL_OPTION, - demo_hash: str = DEMO_HASH_OPTION, - data_preparation_mlcube: int = DATA_PREP_OPTION, - reference_model_mlcube: int = MODEL_OPTION, - evaluator_mlcube: int = EVAL_OPTION + name: str = typer.Option(None, "--name", "-n", help=NAME_HELP), + description: str = typer.Option(None, "--description", "-d", help=DESC_HELP), + docs_url: str = typer.Option(None, "--docs-url", "-u", help=DOCS_HELP), + demo_url: str = typer.Option(None,"--demo-url",help=DEMO_URL_HELP), + demo_hash: str = typer.Option(None, "--demo-hash", help=DEMO_HASH_HELP), + data_preparation_mlcube: int = typer.Option( + None, "--data-preparation-mlcube", "-p", help=DATA_PREP_HELP + ), + reference_model_mlcube: int = typer.Option( + None, "--reference-model-mlcube", "-m", help=MODEL_HELP + ), + evaluator_mlcube: int = typer.Option( + None, "--evaluator-mlcube", "-e", help=EVAL_HELP + ) ): """Edits a benchmark""" benchmark_info = { From 0d55e38300758d209753ea47c54586ae645af17e Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 10 May 2023 11:53:06 -0500 Subject: [PATCH 41/52] Adjust edit command logic --- cli/medperf/commands/edit.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cli/medperf/commands/edit.py b/cli/medperf/commands/edit.py index c50456ad8..fc4e7ebb2 100644 --- a/cli/medperf/commands/edit.py +++ b/cli/medperf/commands/edit.py @@ -21,7 +21,9 @@ def __init__(self, entity_class, id, fields): self.fields = fields def prepare(self): - self.entity = self.entity_class(self.id) + self.entity = self.entity_class.get(self.id) + # Filter out empty fields + self.fields = {k: v for k, v in self.fields.items() if v is not None} def validate(self): if not isinstance(self.entity, Updatable): @@ -29,8 +31,9 @@ def validate(self): def edit(self): entity = self.entity - entity.edit(self.fields) - entity.write() + entity.edit(**self.fields) if isinstance(entity, Updatable) and entity.is_registered: - entity.update() \ No newline at end of file + entity.update() + + entity.write() \ No newline at end of file From a71a3d2caac34a10f6c3252f7397cb6831682a35 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 10 May 2023 17:49:49 -0500 Subject: [PATCH 42/52] Fix production keyword to operation --- cli/medperf/entities/benchmark.py | 9 +-- cli/medperf/entities/cube.py | 98 +++++++++++++++--------------- cli/medperf/entities/dataset.py | 87 +++++++++++++------------- cli/medperf/tests/mocks/cube.py | 2 +- cli/medperf/tests/mocks/dataset.py | 2 +- 5 files changed, 100 insertions(+), 98 deletions(-) diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index b9b2b8c09..121560921 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -296,6 +296,7 @@ def __validate_edit(self, new_bmk: "Benchmark"): # Fields that can no longer be modified while in production production_inmutable_fields = { "name", + "description" "demo_dataset_tarball_hash", "demo_dataset_generated_uid", "data_preparation_mlcube", @@ -303,8 +304,8 @@ def __validate_edit(self, new_bmk: "Benchmark"): "data_evaluator_mlcube" } - if old_bmk.state == "PRODCUTION": - inmutable_fields = inmutable_fields.join(production_inmutable_fields) + if old_bmk.state == "OPERATION": + inmutable_fields = inmutable_fields.union(production_inmutable_fields) bmk_diffs = DeepDiff(new_bmk.todict(), old_bmk.todict()) updated_fields = set(bmk_diffs.affected_root_keys) @@ -315,7 +316,7 @@ def __validate_edit(self, new_bmk: "Benchmark"): fields_msg = ", ".join(updated_inmutable_fields) msg = (f"The following fields can't be directly edited: "\ + fields_msg \ - + ". For these changes, a new Dataset is required") + + ". For these changes, a new Benchmark is required") raise InvalidArgumentError(msg) def update(self): @@ -324,7 +325,7 @@ def update(self): if not self.is_registered: raise MedperfException("Can't update an unregistered benchmark") body = self.todict() - config.comms.update_benchmark(body) + config.comms.update_benchmark(self.id, body) def todict(self) -> dict: """Dictionary representation of the benchmark instance diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index f02aab540..428bf629a 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -2,6 +2,7 @@ import yaml import pexpect import logging +from deepdiff import DeepDiff from typing import List, Dict, Optional, Union from pydantic import Field from pathlib import Path @@ -63,52 +64,6 @@ def __init__(self, *args, **kwargs): if self.git_parameters_url: self.params_path = os.path.join(path, config.params_filename) - @classmethod - def __validate_edit(cls, old_cube: "Cube", new_cube: "Cube"): - """Validates that an edit is valid given the changes made - - Args: - old_cube (Cube): The old version of the MLCube - new_cube (Cube): The new version of the same MLCube - - Raises: - InvalidEntityError: The changes created an invalid entity configuration - InvalidArugmentError: The changed fields are not mutable - """ - # Fields that shouldn't be modified directly by the user - inmutable_fields = {"id",} - - # Fields that can no longer be modified while in production - production_inmutable_fields = { - "mlcube_hash", - "parameters_hash", - "image_tarball_hash", - "additional_files_tarball_hash" - } - - if old_cube.state == "PRODUCTION": - inmutable_fields = inmutable_fields.join(production_inmutable_fields) - - updated_field_values = set(new_cube.items()) - set(old_cube.items()) - updated_fields = {field for field, _ in updated_field_values} - - # Download any new files - new_cube.download(updated_fields) - - updated_inmutable_fields = updated_fields.intersection(production_inmutable_fields) - - if not new_cube.valid(): - msg = "Invalid MLCube configuration" - raise InvalidEntityError(msg) - - - if len(updated_inmutable_fields): - fields_msg = ", ".join(updated_inmutable_fields) - msg = (f"The following fields can't be directly edited: "\ - + fields_msg \ - + ". For these changes, a new MLCube is required") - raise InvalidArgumentError(msg) - @classmethod def all(cls, local_only: bool = False, filters: dict = {}) -> List["Cube"]: """Class method for retrieving all retrievable MLCubes @@ -395,9 +350,56 @@ def edit(self, **kwargs): data.update(kwargs) new_cube = Cube(**data) - Cube.__validate_edit(self, new_cube) + self.__validate_edit(new_cube) - self.__dict__ = new_cube.__dict__ + self.__dict__.update(**new_cube.__dict__) + + def __validate_edit(self, new_cube: "Cube"): + """Validates that an edit is valid given the changes made + + Args: + new_cube (Cube): The new version of the same MLCube + + Raises: + InvalidEntityError: The changes created an invalid entity configuration + InvalidArugmentError: The changed fields are not mutable + """ + old_cube = self + # Fields that shouldn't be modified directly by the user + inmutable_fields = {"id",} + + # Fields that can no longer be modified while in production + production_inmutable_fields = { + "mlcube_hash", + "parameters_hash", + "image_tarball_hash", + "additional_files_tarball_hash" + } + + + if old_cube.state == "OPERATION": + inmutable_fields = inmutable_fields.union(production_inmutable_fields) + + cube_diffs = DeepDiff(new_cube.todict(), old_cube.todict()) + updated_fields = set(cube_diffs.affected_root_keys) + + # Download any new files + # TODO: Download procedure should also update cube hashes and recheck difference + new_cube.download(updated_fields) + + updated_inmutable_fields = updated_fields.intersection(inmutable_fields) + + if not new_cube.valid(): + msg = "Invalid MLCube configuration" + raise InvalidEntityError(msg) + + + if len(updated_inmutable_fields): + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: "\ + + fields_msg \ + + ". For these changes, a new MLCube is required") + raise InvalidArgumentError(msg) def update(self): """Updates the benchmark on the server diff --git a/cli/medperf/entities/dataset.py b/cli/medperf/entities/dataset.py index 04cc77efc..1d4bf0130 100644 --- a/cli/medperf/entities/dataset.py +++ b/cli/medperf/entities/dataset.py @@ -1,6 +1,7 @@ import os import yaml import logging +from deepdiff import DeepDiff from pydantic import Field, validator from typing import List, Optional, Union @@ -69,48 +70,6 @@ def __init__(self, *args, **kwargs): if self.separate_labels: self.labels_path = os.path.join(self.path, "labels") - @classmethod - def __validate_edit(cls, old_dset: "Dataset", new_dset: "Dataset"): - """Determines if an update is valid given the changes made - - Args: - old_dset (Dataset): The old version of the dataset - new_dset (Dataset): The updated version of the same dataset - - Raises: - InvalidArugmentError: The changed fields are not mutable - """ - - # Fields that shouldn't be modified directly by the user - inmutable_fields = { - "id", - "input_data_hash", - "generated_uid", - "separate_labels", - "generated_metadata", - "data_preparation_mlcube", - } - - # Fields that can no longer be modified while in production - production_inmutable_fields = { - "name", - "split_seed" - } - - if old_dset.state == "PRODUCTION": - inmutable_fields = inmutable_fields.join(production_inmutable_fields) - - updated_field_values = set(new_dset.items()) - set(old_dset.items()) - updated_fields = {field for field, _ in updated_field_values} - updated_inmutable_fields = updated_fields.intersection(inmutable_fields) - - if len(updated_inmutable_fields): - fields_msg = ", ".join(updated_inmutable_fields) - msg = (f"The following fields can't be directly edited: " \ - + fields_msg \ - + ". For these changes, a new dataset is required") - raise InvalidArgumentError(msg) - def todict(self): return self.extended_dict() @@ -266,9 +225,49 @@ def edit(self, **kwargs): data.update(kwargs) new_dset = Dataset(**data) - Dataset.__validate_edit(self, new_dset) + self.__validate_edit(new_dset) + + self.__dict__.update(**new_dset.__dict__) + + def __validate_edit(self, new_dset: "Dataset"): + """Determines if an update is valid given the changes made + + Args: + new_dset (Dataset): The updated version of the same dataset + + Raises: + InvalidArugmentError: The changed fields are not mutable + """ + old_dset = self + # Fields that shouldn't be modified directly by the user + inmutable_fields = { + "id", + "input_data_hash", + "generated_uid", + "separate_labels", + "generated_metadata", + "data_preparation_mlcube", + } + + # Fields that can no longer be modified while in production + production_inmutable_fields = { + "name", + "split_seed" + } - self.__dict__ = new_dset.__dict__ + if old_dset.state == "OPERATION": + inmutable_fields = inmutable_fields.union(production_inmutable_fields) + + dset_diffs = DeepDiff(new_dset.todict(), old_dset.todict()) + updated_fields = set(dset_diffs.affected_root_keys) + updated_inmutable_fields = updated_fields.intersection(inmutable_fields) + + if len(updated_inmutable_fields): + fields_msg = ", ".join(updated_inmutable_fields) + msg = (f"The following fields can't be directly edited: " \ + + fields_msg \ + + ". For these changes, a new Dataset is required") + raise InvalidArgumentError(msg) def update(self): """Updates the benchmark on the server diff --git a/cli/medperf/tests/mocks/cube.py b/cli/medperf/tests/mocks/cube.py index 0ad5ba326..885c388e2 100644 --- a/cli/medperf/tests/mocks/cube.py +++ b/cli/medperf/tests/mocks/cube.py @@ -38,4 +38,4 @@ class TestCube(Cube): str ] = "https://test.com/additional_files.tar.gz" additional_files_tarball_hash: Optional[str] = EMPTY_FILE_HASH - state: str = "PRODUCTION" + state: str = "OPERATION" diff --git a/cli/medperf/tests/mocks/dataset.py b/cli/medperf/tests/mocks/dataset.py index 9fb79d5ff..7a3d7528d 100644 --- a/cli/medperf/tests/mocks/dataset.py +++ b/cli/medperf/tests/mocks/dataset.py @@ -12,4 +12,4 @@ class TestDataset(Dataset): generated_uid: str = "generated_uid" generated_metadata: dict = {} status: Status = Status.APPROVED.value - state: str = "PRODUCTION" + state: str = "OPERATION" From e3405242a4f4660856aa4da74e3189c332936ffd Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 10 May 2023 17:52:42 -0500 Subject: [PATCH 43/52] Implement rest update methods --- cli/medperf/comms/interface.py | 24 ++++++++++++++++++++++++ cli/medperf/comms/rest.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/cli/medperf/comms/interface.py b/cli/medperf/comms/interface.py index f934435d5..17eab4616 100644 --- a/cli/medperf/comms/interface.py +++ b/cli/medperf/comms/interface.py @@ -133,6 +133,14 @@ def upload_benchmark(self, benchmark_dict: dict) -> int: int: UID of newly created benchmark """ + @abstractmethod + def update_benchmark(self, id: int, benchmark_dict: dict): + """Updates the benchmark with the given id and the new dictionary + + Args: + benchmark_dict (dict): updated benchmark data + """ + @abstractmethod def upload_mlcube(self, mlcube_body: dict) -> int: """Uploads an MLCube instance to the platform @@ -144,6 +152,14 @@ def upload_mlcube(self, mlcube_body: dict) -> int: int: id of the created mlcube instance on the platform """ + @abstractmethod + def update_mlcube(self, id: int, mlcube_dict: dict): + """Updates the mlcube with the given id and the new dictionary + + Args: + mlcube_dict (dict): updated mlcube data + """ + @abstractmethod def get_datasets(self) -> List[dict]: """Retrieves all datasets in the platform @@ -182,6 +198,14 @@ def upload_dataset(self, reg_dict: dict) -> int: int: id of the created dataset registration. """ + @abstractmethod + def update_dataset(self, id: int, dataset_dict: dict): + """Updates the dataset with the given id and the new dictionary + + Args: + dataset_dict (dict): updated dataset data + """ + @abstractmethod def get_results(self) -> List[dict]: """Retrieves all results diff --git a/cli/medperf/comms/rest.py b/cli/medperf/comms/rest.py index 8da1d766b..0ccb3c4d5 100644 --- a/cli/medperf/comms/rest.py +++ b/cli/medperf/comms/rest.py @@ -287,6 +287,17 @@ def upload_benchmark(self, benchmark_dict: dict) -> int: raise CommunicationRetrievalError("Could not upload benchmark") return res.json() + def update_benchmark(self, id: int, benchmark_dict: dict): + """Updates the benchmark with the given id and the new dictionary + + Args: + benchmark_dict (dict): updated benchmark data + """ + res = self.__auth_put(f"{self.server_url}/benchmarks/{id}/", json=benchmark_dict) + if res.status_code != 200: + log_response_error(res) + raise CommunicationRequestError(f"Could not update benchmark") + def upload_mlcube(self, mlcube_body: dict) -> int: """Uploads an MLCube instance to the platform @@ -302,6 +313,17 @@ def upload_mlcube(self, mlcube_body: dict) -> int: raise CommunicationRetrievalError("Could not upload the mlcube") return res.json() + def update_mlcube(self, id: int, mlcube_dict: dict): + """Updates the mlcube with the given id and the new dictionary + + Args: + mlcube_dict (dict): updated mlcube data + """ + res = self.__auth_put(f"{self.server_url}/mlcubes/{id}/", json=mlcube_dict) + if res.status_code != 200: + log_response_error(res) + raise CommunicationRequestError(f"Could not update mlcube") + def get_datasets(self) -> List[dict]: """Retrieves all datasets in the platform @@ -352,6 +374,17 @@ def upload_dataset(self, reg_dict: dict) -> int: raise CommunicationRequestError("Could not upload the dataset") return res.json() + def update_dataset(self, id: int, dataset_dict: dict): + """Updates the dataset with the given id and the new dictionary + + Args: + dataset_dict (dict): updated dataset data + """ + res = self.__auth_put(f"{self.server_url}/datasets/{id}/", json=dataset_dict) + if res.status_code != 200: + log_response_error(res) + raise CommunicationRequestError(f"Could not update dataset") + def get_results(self) -> List[dict]: """Retrieves all results From f9e4f440aca7045ed1b12901c7666d4ee0731c76 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 15 May 2023 10:36:52 -0500 Subject: [PATCH 44/52] Provide edit commands --- cli/medperf/commands/benchmark/benchmark.py | 4 +- cli/medperf/commands/dataset/dataset.py | 25 +++--- cli/medperf/commands/mlcube/mlcube.py | 84 ++++++++++++++++++--- cli/medperf/entities/dataset.py | 6 +- 4 files changed, 94 insertions(+), 25 deletions(-) diff --git a/cli/medperf/commands/benchmark/benchmark.py b/cli/medperf/commands/benchmark/benchmark.py index 1bdd5dde9..6904f5811 100644 --- a/cli/medperf/commands/benchmark/benchmark.py +++ b/cli/medperf/commands/benchmark/benchmark.py @@ -91,7 +91,8 @@ def edit( ), evaluator_mlcube: int = typer.Option( None, "--evaluator-mlcube", "-e", help=EVAL_HELP - ) + ), + is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments") ): """Edits a benchmark""" benchmark_info = { @@ -103,6 +104,7 @@ def edit( "data_preparation_mlcube": data_preparation_mlcube, "reference_model_mlcube": reference_model_mlcube, "data_evaluator_mlcube": evaluator_mlcube, + "is_valid": is_valid, } EntityEdit.run(Benchmark, entity_id, benchmark_info) cleanup() diff --git a/cli/medperf/commands/dataset/dataset.py b/cli/medperf/commands/dataset/dataset.py index 69dca0c5c..6e038560c 100644 --- a/cli/medperf/commands/dataset/dataset.py +++ b/cli/medperf/commands/dataset/dataset.py @@ -11,13 +11,10 @@ from medperf.commands.dataset.submit import DatasetRegistration from medperf.commands.dataset.associate import AssociateDataset -NAME_OPTION = typer.Option(..., "--name", help="Name of the dataset") -DESC_OPTION = typer.Option( - ..., "--description", help="Description of the dataset" -) -LOC_OPTION = typer.Option( - ..., "--location", help="Location or Institution the data belongs to" -) +NAME_HELP = "Name of the dataset" +DESC_HELP = "Description of the dataset" +LOC_HELP = "Location or Institution the data belongs to" +LOC_OPTION = typer.Option(..., "--location", help=LOC_HELP) app = typer.Typer() @@ -52,9 +49,9 @@ def create( labels_path: str = typer.Option( ..., "--labels_path", "-l", help="Labels file location" ), - name: str = NAME_OPTION, - description: str = DESC_OPTION, - location: str = LOC_OPTION + name: str = typer.Option(..., "--name", help=NAME_HELP), + description: str = typer.Option(..., "--description", help=DESC_HELP), + location: str = typer.Option(..., "--location", help=LOC_HELP) ): """Runs the Data preparation step for a specified benchmark and raw dataset """ @@ -96,15 +93,17 @@ def register( @clean_except def edit( entity_id: int = typer.Argument(..., help="Dataset ID"), - name: str = NAME_OPTION, - description: str = DESC_OPTION, - location: str = LOC_OPTION + name: str = typer.Option(None, "--name", help=NAME_HELP), + description: str = typer.Option(None, "--description", help=DESC_HELP), + location: str = typer.Option(None, "--location", help=LOC_HELP), + is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments") ): """Edits a Dataset""" dset_info = { "name": name, "description": description, "location": location, + "is_valid": is_valid, } EntityEdit.run(Dataset, entity_id, dset_info) config.ui.print("✅ Done!") diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index 2a8852168..960f45feb 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -7,12 +7,23 @@ from medperf.entities.cube import Cube from medperf.commands.list import EntityList from medperf.commands.view import EntityView +from medperf.commands.edit import EntityEdit from medperf.commands.mlcube.create import CreateCube from medperf.commands.mlcube.submit import SubmitCube from medperf.commands.mlcube.associate import AssociateCube app = typer.Typer() +NAME_HELP = "Name of the mlcube" +MLCUBE_HELP = "Identifier to download the mlcube file. See the description above" +MLCUBE_HASH_HELP = "SHA1 of mlcube file" +PARAMS_HELP = "Identifier to download the parameters file. See the description above" +PARAMS_HASH_HELP = "SHA1 of parameters file" +ADD_HELP = "Identifier to download the additional files tarball. See the description above" +ADD_HASH_HELP = "SHA1 of additional file" +IMG_HELP = "Identifier to download the image file. See the description above" +IMG_HASH_HELP = "SHA1 of image file" + @app.command("ls") @clean_except @@ -53,39 +64,39 @@ def create( @app.command("submit") @clean_except def submit( - name: str = typer.Option(..., "--name", "-n", help="Name of the mlcube"), + name: str = typer.Option(..., "--name", "-n", help=NAME_HELP), mlcube_file: str = typer.Option( ..., "--mlcube-file", "-m", - help="Identifier to download the mlcube file. See the description above", + help=MLCUBE_HELP, ), - mlcube_hash: str = typer.Option("", "--mlcube-hash", help="SHA1 of mlcube file"), + mlcube_hash: str = typer.Option("", "--mlcube-hash", help=MLCUBE_HASH_HELP), parameters_file: str = typer.Option( "", "--parameters-file", "-p", - help="Identifier to download the parameters file. See the description above", + help=PARAMS_HELP, ), parameters_hash: str = typer.Option( - "", "--parameters-hash", help="SHA1 of parameters file" + "", "--parameters-hash", help=PARAMS_HASH_HELP ), additional_file: str = typer.Option( "", "--additional-file", "-a", - help="Identifier to download the additional files tarball. See the description above", + help=ADD_HELP, ), additional_hash: str = typer.Option( - "", "--additional-hash", help="SHA1 of additional file" + "", "--additional-hash", help=ADD_HASH_HELP ), image_file: str = typer.Option( "", "--image-file", "-i", - help="Identifier to download the image file. See the description above", + help=IMG_HELP, ), - image_hash: str = typer.Option("", "--image-hash", help="SHA1 of image file"), + image_hash: str = typer.Option("", "--image-hash", help=IMG_HASH_HELP), ): """Submits a new cube to the platform.\n The following assets:\n @@ -117,6 +128,61 @@ def submit( config.ui.print("✅ Done!") +@app.command("edit") +@clean_except +def edit( + entity_id: int = typer.Argument(..., help="Dataset ID"), + name: str = typer.Option(None, "--name", "-n", help=NAME_HELP), + mlcube_file: str = typer.Option( + None, + "--mlcube-file", + "-m", + help=MLCUBE_HELP, + ), + mlcube_hash: str = typer.Option(None, "--mlcube-hash", help=MLCUBE_HASH_HELP), + parameters_file: str = typer.Option( + None, + "--parameters-file", + "-p", + help=PARAMS_HELP, + ), + parameters_hash: str = typer.Option( + None, "--parameters-hash", help=PARAMS_HASH_HELP + ), + additional_file: str = typer.Option( + None, + "--additional-file", + "-a", + help=ADD_HELP, + ), + additional_hash: str = typer.Option( + None, "--additional-hash", help=ADD_HASH_HELP + ), + image_file: str = typer.Option( + None, + "--image-file", + "-i", + help=IMG_HELP, + ), + image_hash: str = typer.Option(None, "--image-hash", help=IMG_HASH_HELP), + is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags an MLCube valid/invalid. Invalid MLCubes can't be used for experiments") +): + """Edits an MLCube""" + mlcube_info = { + "name": name, + "git_mlcube_url": mlcube_file, + "git_mlcube_hash": mlcube_hash, + "git_parameters_url": parameters_file, + "parameters_hash": parameters_hash, + "image_tarball_url": image_file, + "image_tarball_hash": image_hash, + "additional_files_tarball_url": additional_file, + "additional_files_tarball_hash": additional_hash, + } + EntityEdit.run(Cube, entity_id, mlcube_info) + config.ui.print("✅ Done!") + + @app.command("associate") @clean_except def associate( diff --git a/cli/medperf/entities/dataset.py b/cli/medperf/entities/dataset.py index 1d4bf0130..1e0b87ecd 100644 --- a/cli/medperf/entities/dataset.py +++ b/cli/medperf/entities/dataset.py @@ -252,7 +252,9 @@ def __validate_edit(self, new_dset: "Dataset"): # Fields that can no longer be modified while in production production_inmutable_fields = { "name", - "split_seed" + "split_seed", + "description", + "location" } if old_dset.state == "OPERATION": @@ -275,7 +277,7 @@ def update(self): if not self.is_registered: raise MedperfException("Can't update an unregistered dataset") body = self.todict() - config.comms.update_dataset(body) + config.comms.update_dataset(self.id, body) @classmethod def __get_local_dict(cls, data_uid): From 5424a7e89b9baa34806ebe6b77db7155925a54cc Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 15 May 2023 10:37:03 -0500 Subject: [PATCH 45/52] Add more descriptive error --- cli/medperf/comms/rest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/medperf/comms/rest.py b/cli/medperf/comms/rest.py index 0ccb3c4d5..6bdee4512 100644 --- a/cli/medperf/comms/rest.py +++ b/cli/medperf/comms/rest.py @@ -296,7 +296,7 @@ def update_benchmark(self, id: int, benchmark_dict: dict): res = self.__auth_put(f"{self.server_url}/benchmarks/{id}/", json=benchmark_dict) if res.status_code != 200: log_response_error(res) - raise CommunicationRequestError(f"Could not update benchmark") + raise CommunicationRequestError(f"Could not update benchmark: {res.text}") def upload_mlcube(self, mlcube_body: dict) -> int: """Uploads an MLCube instance to the platform @@ -322,7 +322,7 @@ def update_mlcube(self, id: int, mlcube_dict: dict): res = self.__auth_put(f"{self.server_url}/mlcubes/{id}/", json=mlcube_dict) if res.status_code != 200: log_response_error(res) - raise CommunicationRequestError(f"Could not update mlcube") + raise CommunicationRequestError(f"Could not update mlcube: {res.text}") def get_datasets(self) -> List[dict]: """Retrieves all datasets in the platform @@ -383,7 +383,7 @@ def update_dataset(self, id: int, dataset_dict: dict): res = self.__auth_put(f"{self.server_url}/datasets/{id}/", json=dataset_dict) if res.status_code != 200: log_response_error(res) - raise CommunicationRequestError(f"Could not update dataset") + raise CommunicationRequestError(f"Could not update dataset: {res.text}") def get_results(self) -> List[dict]: """Retrieves all results From 5f32e50da2060d9b7def7dd1a4521eec19eec773 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Tue, 16 May 2023 15:02:21 -0500 Subject: [PATCH 46/52] Abstract field-error dict formatting --- cli/medperf/utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index 60e44141f..6962bd83a 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -493,6 +493,25 @@ def log_response_error(res, warn=False): logging_method(res.content) +def format_errors_dict(errors_dict: dict): + """Reformats the error details from a field-error(s) dictionary into a human-readable string for printing""" + error_msg = "" + for field, errors in errors_dict.items(): + error_msg += "\n" + if isinstance(field, tuple): + field = field[0] + error_msg += f"- {field}: " + if len(errors) == 1: + # If a single error for a field is given, don't create a sublist + error_msg += errors[0] + else: + # Create a sublist otherwise + for e_msg in errors: + error_msg += "\n" + error_msg += f"\t- {e_msg}" + return error_msg + + def get_cube_image_name(cube_path: str) -> str: """Retrieves the singularity image name of the mlcube by reading its mlcube.yaml file""" cube_config_path = os.path.join(cube_path, config.cube_filename) From 3951af889312c9f3ab0e04092664100cd1d25ecd Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Tue, 16 May 2023 15:02:36 -0500 Subject: [PATCH 47/52] Reformat errors dictionary for printing --- cli/medperf/comms/rest.py | 55 ++++++++++++++++++++++----------- cli/medperf/entities/schemas.py | 14 ++------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/cli/medperf/comms/rest.py b/cli/medperf/comms/rest.py index 8da1d766b..3d4c85d71 100644 --- a/cli/medperf/comms/rest.py +++ b/cli/medperf/comms/rest.py @@ -9,6 +9,7 @@ read_credentials, sanitize_json, log_response_error, + format_errors_dict, ) from medperf.exceptions import ( CommunicationError, @@ -58,8 +59,9 @@ def login(self, user: str, pwd: str): res = self.__req(f"{self.server_url}/auth-token/", requests.post, json=body) if res.status_code != 200: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationAuthenticationError( - "Unable to authenticate user with provided credentials" + f"Unable to authenticate user with provided credentials: {details}" ) else: self.token = res.json()["token"] @@ -74,7 +76,8 @@ def change_password(self, pwd: str): res = self.__auth_post(f"{self.server_url}/me/password/", json=body) if res.status_code != 200: log_response_error(res) - raise CommunicationRequestError("Unable to change the current password") + details = format_errors_dict(res.json()) + raise CommunicationRequestError(f"Unable to change the current password: {details}") def authenticate(self): token = read_credentials() @@ -151,14 +154,16 @@ def __get_list( if res.status_code != 200: if not binary_reduction: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationRetrievalError( - "there was an error retrieving the current list." + f"there was an error retrieving the current list: {details}" ) log_response_error(res, warn=True) + details = format_errors_dict(res.json()) if page_size <= 1: raise CommunicationRetrievalError( - "Could not retrieve list. Minimum page size achieved without success." + f"Could not retrieve list. Minimum page size achieved without success: {details}" ) page_size = page_size // 2 continue @@ -214,7 +219,8 @@ def get_benchmark(self, benchmark_uid: int) -> dict: res = self.__auth_get(f"{self.server_url}/benchmarks/{benchmark_uid}") if res.status_code != 200: log_response_error(res) - raise CommunicationRetrievalError("the specified benchmark doesn't exist") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"the specified benchmark doesn't exist: {details}") return res.json() def get_benchmark_models(self, benchmark_uid: int) -> List[int]: @@ -260,7 +266,8 @@ def get_cube_metadata(self, cube_uid: int) -> dict: res = self.__auth_get(f"{self.server_url}/mlcubes/{cube_uid}/") if res.status_code != 200: log_response_error(res) - raise CommunicationRetrievalError("the specified cube doesn't exist") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"the specified cube doesn't exist {details}") return res.json() def get_user_cubes(self) -> List[dict]: @@ -284,7 +291,8 @@ def upload_benchmark(self, benchmark_dict: dict) -> int: res = self.__auth_post(f"{self.server_url}/benchmarks/", json=benchmark_dict) if res.status_code != 201: log_response_error(res) - raise CommunicationRetrievalError("Could not upload benchmark") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"Could not upload benchmark: {details}") return res.json() def upload_mlcube(self, mlcube_body: dict) -> int: @@ -299,7 +307,8 @@ def upload_mlcube(self, mlcube_body: dict) -> int: res = self.__auth_post(f"{self.server_url}/mlcubes/", json=mlcube_body) if res.status_code != 201: log_response_error(res) - raise CommunicationRetrievalError("Could not upload the mlcube") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"Could not upload the mlcube: {details}") return res.json() def get_datasets(self) -> List[dict]: @@ -323,8 +332,9 @@ def get_dataset(self, dset_uid: int) -> dict: res = self.__auth_get(f"{self.server_url}/datasets/{dset_uid}/") if res.status_code != 200: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationRetrievalError( - "Could not retrieve the specified dataset from server" + f"Could not retrieve the specified dataset from server: {details}" ) return res.json() @@ -349,7 +359,8 @@ def upload_dataset(self, reg_dict: dict) -> int: res = self.__auth_post(f"{self.server_url}/datasets/", json=reg_dict) if res.status_code != 201: log_response_error(res) - raise CommunicationRequestError("Could not upload the dataset") + details = format_errors_dict(res.json()) + raise CommunicationRequestError(f"Could not upload the dataset: {details}") return res.json() def get_results(self) -> List[dict]: @@ -361,7 +372,8 @@ def get_results(self) -> List[dict]: res = self.__get_list(f"{self.server_url}/results") if res.status_code != 200: log_response_error(res) - raise CommunicationRetrievalError("Could not retrieve results") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"Could not retrieve results: {details}") return res.json() def get_result(self, result_uid: int) -> dict: @@ -376,7 +388,8 @@ def get_result(self, result_uid: int) -> dict: res = self.__auth_get(f"{self.server_url}/results/{result_uid}/") if res.status_code != 200: log_response_error(res) - raise CommunicationRetrievalError("Could not retrieve the specified result") + details = format_errors_dict(res.json()) + raise CommunicationRetrievalError(f"Could not retrieve the specified result: {details}") return res.json() def get_user_results(self) -> dict: @@ -414,7 +427,8 @@ def upload_result(self, results_dict: dict) -> int: res = self.__auth_post(f"{self.server_url}/results/", json=results_dict) if res.status_code != 201: log_response_error(res) - raise CommunicationRequestError("Could not upload the results") + details = format_errors_dict(res.json()) + raise CommunicationRequestError(f"Could not upload the results: {details}") return res.json() def associate_dset(self, data_uid: int, benchmark_uid: int, metadata: dict = {}): @@ -434,7 +448,8 @@ def associate_dset(self, data_uid: int, benchmark_uid: int, metadata: dict = {}) res = self.__auth_post(f"{self.server_url}/datasets/benchmarks/", json=data) if res.status_code != 201: log_response_error(res) - raise CommunicationRequestError("Could not associate dataset to benchmark") + details = format_errors_dict(res.json()) + raise CommunicationRequestError(f"Could not associate dataset to benchmark: {details}") def associate_cube(self, cube_uid: int, benchmark_uid: int, metadata: dict = {}): """Create an MLCube-Benchmark association @@ -453,7 +468,8 @@ def associate_cube(self, cube_uid: int, benchmark_uid: int, metadata: dict = {}) res = self.__auth_post(f"{self.server_url}/mlcubes/benchmarks/", json=data) if res.status_code != 201: log_response_error(res) - raise CommunicationRequestError("Could not associate mlcube to benchmark") + details = format_errors_dict(res.json()) + raise CommunicationRequestError(f"Could not associate mlcube to benchmark: {details}") def set_dataset_association_approval( self, benchmark_uid: int, dataset_uid: int, status: str @@ -469,8 +485,9 @@ def set_dataset_association_approval( res = self.__set_approval_status(url, status) if res.status_code != 200: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationRequestError( - f"Could not approve association between dataset {dataset_uid} and benchmark {benchmark_uid}" + f"Could not approve association between dataset {dataset_uid} and benchmark {benchmark_uid}: {details}" ) def set_mlcube_association_approval( @@ -487,8 +504,9 @@ def set_mlcube_association_approval( res = self.__set_approval_status(url, status) if res.status_code != 200: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationRequestError( - f"Could not approve association between mlcube {mlcube_uid} and benchmark {benchmark_uid}" + f"Could not approve association between mlcube {mlcube_uid} and benchmark {benchmark_uid}: {details}" ) def get_datasets_associations(self) -> List[dict]: @@ -524,6 +542,7 @@ def set_mlcube_association_priority( res = self.__auth_put(url, json=data,) if res.status_code != 200: log_response_error(res) + details = format_errors_dict(res.json()) raise CommunicationRequestError( - f"Could not set the priority of mlcube {mlcube_uid} within the benchmark {benchmark_uid}" + f"Could not set the priority of mlcube {mlcube_uid} within the benchmark {benchmark_uid}: {details}" ) diff --git a/cli/medperf/entities/schemas.py b/cli/medperf/entities/schemas.py index 9dccaa8d3..27abe0ee5 100644 --- a/cli/medperf/entities/schemas.py +++ b/cli/medperf/entities/schemas.py @@ -5,6 +5,7 @@ from medperf.enums import Status from medperf.exceptions import MedperfException +from medperf.utils import format_errors_dict class MedperfBaseSchema(BaseModel): @@ -22,18 +23,7 @@ def __init__(self, *args, **kwargs): errors_dict[field].append(msg) error_msg = "Field Validation Error:" - for field, errors in errors_dict.items(): - error_msg += "\n" - field = field[0] - error_msg += f"- {field}: " - if len(errors) == 1: - # If a single error for a field is given, don't create a sublist - error_msg += errors[0] - else: - # Create a sublist otherwise - for e_msg in errors: - error_msg += "\n" - error_msg += f"\t- {e_msg}" + error_msg += format_errors_dict(errors_dict) raise MedperfException(error_msg) From e2fd99756825f162fb3d03be7513c4ffa0c4cf8e Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 31 May 2023 12:30:38 -0500 Subject: [PATCH 48/52] Add mlcube update logic --- cli/medperf/commands/mlcube/mlcube.py | 32 ++++++------ .../comms/entity_resources/resources.py | 33 ++++++++---- cli/medperf/entities/cube.py | 50 ++++++++++++------- 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index 9c957f74f..1244e2ae7 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -18,9 +18,11 @@ MLCUBE_HASH_HELP = "SHA1 of mlcube file" PARAMS_HELP = "Identifier to download the parameters file. See the description above" PARAMS_HASH_HELP = "SHA1 of parameters file" -ADD_HELP = "Identifier to download the additional files tarball. See the description above" +ADD_HELP = ( + "Identifier to download the additional files tarball. See the description above" +) ADD_HASH_HELP = "SHA1 of additional file" -IMG_HELP = "Identifier to download the image file. See the description above" +IMG_HELP = "Identifier to download the image file. See the description above" IMG_HASH_HELP = "SHA1 of image file" @@ -77,18 +79,14 @@ def submit( "-p", help=PARAMS_HELP, ), - parameters_hash: str = typer.Option( - "", "--parameters-hash", help=PARAMS_HASH_HELP - ), + parameters_hash: str = typer.Option("", "--parameters-hash", help=PARAMS_HASH_HELP), additional_file: str = typer.Option( "", "--additional-file", "-a", help=ADD_HELP, ), - additional_hash: str = typer.Option( - "", "--additional-hash", help=ADD_HASH_HELP - ), + additional_hash: str = typer.Option("", "--additional-hash", help=ADD_HASH_HELP), image_file: str = typer.Option( "", "--image-file", @@ -153,9 +151,7 @@ def edit( "-a", help=ADD_HELP, ), - additional_hash: str = typer.Option( - None, "--additional-hash", help=ADD_HASH_HELP - ), + additional_hash: str = typer.Option(None, "--additional-hash", help=ADD_HASH_HELP), image_file: str = typer.Option( None, "--image-file", @@ -163,7 +159,11 @@ def edit( help=IMG_HELP, ), image_hash: str = typer.Option(None, "--image-hash", help=IMG_HASH_HELP), - is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags an MLCube valid/invalid. Invalid MLCubes can't be used for experiments") + is_valid: bool = typer.Option( + None, + "--valid/--invalid", + help="Flags an MLCube valid/invalid. Invalid MLCubes can't be used for experiments", + ), ): """Edits an MLCube""" mlcube_info = { @@ -176,6 +176,7 @@ def edit( "image_tarball_hash": image_hash, "additional_files_tarball_url": additional_file, "additional_files_tarball_hash": additional_hash, + "is_valid": is_valid, } EntityEdit.run(Cube, entity_id, mlcube_info) config.ui.print("✅ Done!") @@ -188,7 +189,9 @@ def associate( model_uid: int = typer.Option(..., "--model_uid", "-m", help="Model UID"), approval: bool = typer.Option(False, "-y", help="Skip approval step"), no_cache: bool = typer.Option( - False, "--no-cache", help="Execute the test even if results already exist", + False, + "--no-cache", + help="Execute the test even if results already exist", ), ): """Associates an MLCube to a benchmark""" @@ -221,6 +224,5 @@ def view( help="Output file to store contents. If not provided, the output will be displayed", ), ): - """Displays the information of one or more mlcubes - """ + """Displays the information of one or more mlcubes""" EntityView.run(entity_id, Cube, format, local, mine, output) diff --git a/cli/medperf/comms/entity_resources/resources.py b/cli/medperf/comms/entity_resources/resources.py index e0493797f..b88b674f0 100644 --- a/cli/medperf/comms/entity_resources/resources.py +++ b/cli/medperf/comms/entity_resources/resources.py @@ -26,7 +26,9 @@ from .utils import download_resource -def get_cube(url: str, cube_path: str, expected_hash: str = None) -> str: +def get_cube( + url: str, cube_path: str, expected_hash: str = None, force: bool = False +) -> str: """Downloads and writes an mlcube.yaml file. If the hash is provided, the file's integrity will be checked upon download. @@ -34,39 +36,45 @@ def get_cube(url: str, cube_path: str, expected_hash: str = None) -> str: url (str): URL where the mlcube.yaml file can be downloaded. cube_path (str): Cube location. expected_hash (str, optional): expected sha1 hash of the downloaded file + force (bool, optional): Wether to force redownload or not Returns: output_path (str): location where the mlcube.yaml file is stored locally. hash_value (str): The hash of the downloaded file """ output_path = os.path.join(cube_path, config.cube_filename) - if os.path.exists(output_path): + if not force and os.path.exists(output_path): return output_path, expected_hash hash_value = download_resource(url, output_path, expected_hash) return output_path, hash_value -def get_cube_params(url: str, cube_path: str, expected_hash: str = None) -> str: +def get_cube_params( + url: str, cube_path: str, expected_hash: str = None, force: bool = False +) -> str: """Downloads and writes a cube parameters file. If the hash is provided, the file's integrity will be checked upon download. Args: url (str): URL where the parameters.yaml file can be downloaded. cube_path (str): Cube location. - expected_hash (str, optional): expected sha1 hash of the downloaded file + expected_hash (str, Optional): expected sha1 hash of the downloaded file + force (bool, Optional): Wether to force redownload or not Returns: output_path (str): location where the parameters file is stored locally. hash_value (str): The hash of the downloaded file """ output_path = os.path.join(cube_path, config.workspace_path, config.params_filename) - if os.path.exists(output_path): + if not force and os.path.exists(output_path): return output_path, expected_hash hash_value = download_resource(url, output_path, expected_hash) return output_path, hash_value -def get_cube_image(url: str, cube_path: str, hash_value: str = None) -> str: +def get_cube_image( + url: str, cube_path: str, hash_value: str = None, force: bool = False +) -> str: """Retrieves and stores the image file from the server. Stores images on a shared location, and retrieves a cached image by hash if found locally. Creates a symbolic link to the cube storage. @@ -75,6 +83,7 @@ def get_cube_image(url: str, cube_path: str, hash_value: str = None) -> str: url (str): URL where the image file can be downloaded. cube_path (str): Path to cube. hash_value (str, Optional): File hash to store under shared storage. Defaults to None. + force (bool, Optional): Wether to force redownload or not Returns: image_cube_file: Location where the image file is stored locally. @@ -98,7 +107,7 @@ def get_cube_image(url: str, cube_path: str, hash_value: str = None) -> str: shutil.move(tmp_output_path, img_storage) else: img_storage = os.path.join(imgs_storage, hash_value) - if not os.path.exists(img_storage): + if force or not os.path.exists(img_storage): # If image doesn't exist locally, download it normally download_resource(url, img_storage, hash_value) @@ -108,7 +117,7 @@ def get_cube_image(url: str, cube_path: str, hash_value: str = None) -> str: def get_cube_additional( - url: str, cube_path: str, expected_tarball_hash: str = None, + url: str, cube_path: str, expected_tarball_hash: str = None, force: bool = True ) -> str: """Retrieves additional files of an MLCube. The additional files will be in a compressed tarball file. The function will additionally @@ -117,7 +126,8 @@ def get_cube_additional( Args: url (str): URL where the additional_files.tar.gz file can be downloaded. cube_path (str): Cube location. - expected_tarball_hash (str, optional): expected sha1 hash of tarball file + expected_tarball_hash (str, Optional): expected sha1 hash of tarball file + force (bool, Optional): Wether to force redownload or not Returns: tarball_hash (str): The hash of the downloaded tarball file @@ -125,7 +135,10 @@ def get_cube_additional( additional_files_folder = os.path.join(cube_path, config.additional_path) if os.path.exists(additional_files_folder): - return expected_tarball_hash + if force: + shutil.rmtree(additional_files_folder) + else: + return expected_tarball_hash # make sure files are uncompressed while in tmp storage, to avoid any clutter # objects if uncompression fails for some reason. diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index 03d80bf1b..9091dc829 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -178,35 +178,37 @@ def __local_get(cls, cube_uid: Union[str, int]) -> "Cube": cube = cls(**local_meta) return cube - def download_mlcube(self): + def download_mlcube(self, force=False): url = self.git_mlcube_url - path, file_hash = resources.get_cube(url, self.path, self.mlcube_hash) + path, file_hash = resources.get_cube( + url, self.path, self.mlcube_hash, force=force + ) self.cube_path = path self.mlcube_hash = file_hash - def download_parameters(self): + def download_parameters(self, force=False): url = self.git_parameters_url if url: path, file_hash = resources.get_cube_params( - url, self.path, self.parameters_hash + url, self.path, self.parameters_hash, force=force ) self.params_path = path self.parameters_hash = file_hash - def download_additional(self): + def download_additional(self, force=False): url = self.additional_files_tarball_url if url: file_hash = resources.get_cube_additional( - url, self.path, self.additional_files_tarball_hash + url, self.path, self.additional_files_tarball_hash, force=force ) self.additional_files_tarball_hash = file_hash - def download_image(self): + def download_image(self, force=False): url = self.image_tarball_url hash = self.image_tarball_hash if url: - _, local_hash = resources.get_cube_image(url, self.path, hash) + _, local_hash = resources.get_cube_image(url, self.path, hash, force=force) self.image_tarball_hash = local_hash else: # Retrieve image from image registry @@ -308,15 +310,35 @@ def get_default_output(self, task: str, out_key: str, param_key: str = None) -> def edit(self, **kwargs): """Edits a cube with the given property-value pairs""" data = self.todict() + + # Include the updated fields data.update(kwargs) new_cube = Cube(**data) + # If any resource is being updated, download and get the new hash + # Hash difference checking is done between the old and new cube + # According to update policies + if "git_mlcube_url" in kwargs: + new_cube.mlcube_hash = kwargs.get("mlcube_hash", None) + new_cube.download_mlcube(force=True) + if "git_parameters_url" in kwargs: + new_cube.parameters_hash = kwargs.get("parameters_hash", None) + new_cube.download_parameters(force=True) + if "image_tarball_url" in kwargs: + new_cube.image_tarball_hash = kwargs.get("image_tarball_hash", None) + new_cube.download_image(force=True) + if "additional_files_tarball_url" in kwargs: + new_cube.additional_files_tarball_hash = kwargs.get( + "additional_files_tarball_hash", None + ) + new_cube.download_additional(force=True) + self.__validate_edit(new_cube) self.__dict__.update(**new_cube.__dict__) def __validate_edit(self, new_cube: "Cube"): - """Validates that an edit is valid given the changes made + """Ensure an edit is valid given the changes made Args: new_cube (Cube): The new version of the same MLCube @@ -345,16 +367,8 @@ def __validate_edit(self, new_cube: "Cube"): cube_diffs = DeepDiff(new_cube.todict(), old_cube.todict()) updated_fields = set(cube_diffs.affected_root_keys) - # Download any new files - # TODO: Download procedure should also update cube hashes and recheck difference - new_cube.download(updated_fields) - updated_inmutable_fields = updated_fields.intersection(inmutable_fields) - if not new_cube.valid(): - msg = "Invalid MLCube configuration" - raise InvalidEntityError(msg) - if len(updated_inmutable_fields): fields_msg = ", ".join(updated_inmutable_fields) msg = ( @@ -369,7 +383,7 @@ def update(self): if not self.is_registered: raise MedperfException("Can't update an unregistered cube") body = self.todict() - config.comms.update_mlcube(body) + config.comms.update_mlcube(self.id, body) def todict(self) -> Dict: return self.extended_dict() From a0e8d529438bb9eafc1b44b2488cfce6eef90b99 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 31 May 2023 17:16:50 -0500 Subject: [PATCH 49/52] Fix linter issues --- cli/medperf/commands/benchmark/benchmark.py | 30 ++++++++++++--------- cli/medperf/commands/dataset/dataset.py | 21 ++++++++------- cli/medperf/commands/edit.py | 5 ++-- cli/medperf/entities/benchmark.py | 23 ++++++++-------- cli/medperf/entities/cube.py | 2 +- cli/medperf/entities/dataset.py | 23 +++++++--------- cli/medperf/entities/interface.py | 3 +-- 7 files changed, 55 insertions(+), 52 deletions(-) diff --git a/cli/medperf/commands/benchmark/benchmark.py b/cli/medperf/commands/benchmark/benchmark.py index eb2155b5c..cdbc828e7 100644 --- a/cli/medperf/commands/benchmark/benchmark.py +++ b/cli/medperf/commands/benchmark/benchmark.py @@ -45,7 +45,7 @@ def submit( name: str = typer.Option(..., "--name", "-n", help=NAME_HELP), description: str = typer.Option(..., "--description", "-d", help=DESC_HELP), docs_url: str = typer.Option("", "--docs-url", "-u", help=DOCS_HELP), - demo_url: str = typer.Option("","--demo-url",help=DEMO_URL_HELP), + demo_url: str = typer.Option("", "--demo-url", help=DEMO_URL_HELP), demo_hash: str = typer.Option("", "--demo-hash", help=DEMO_HASH_HELP), data_preparation_mlcube: int = typer.Option( ..., "--data-preparation-mlcube", "-p", help=DATA_PREP_HELP @@ -55,7 +55,7 @@ def submit( ), evaluator_mlcube: int = typer.Option( ..., "--evaluator-mlcube", "-e", help=EVAL_HELP - ) + ), ): """Submits a new benchmark to the platform""" benchmark_info = { @@ -79,7 +79,7 @@ def edit( name: str = typer.Option(None, "--name", "-n", help=NAME_HELP), description: str = typer.Option(None, "--description", "-d", help=DESC_HELP), docs_url: str = typer.Option(None, "--docs-url", "-u", help=DOCS_HELP), - demo_url: str = typer.Option(None,"--demo-url",help=DEMO_URL_HELP), + demo_url: str = typer.Option(None, "--demo-url", help=DEMO_URL_HELP), demo_hash: str = typer.Option(None, "--demo-hash", help=DEMO_HASH_HELP), data_preparation_mlcube: int = typer.Option( None, "--data-preparation-mlcube", "-p", help=DATA_PREP_HELP @@ -90,7 +90,11 @@ def edit( evaluator_mlcube: int = typer.Option( None, "--evaluator-mlcube", "-e", help=EVAL_HELP ), - is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments") + is_valid: bool = typer.Option( + None, + "--valid/--invalid", + help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments", + ), ): """Edits a benchmark""" benchmark_info = { @@ -105,7 +109,6 @@ def edit( "is_valid": is_valid, } EntityEdit.run(Benchmark, entity_id, benchmark_info) - cleanup() config.ui.print("✅ Done!") @@ -123,11 +126,12 @@ def associate( ), approval: bool = typer.Option(False, "-y", help="Skip approval step"), no_cache: bool = typer.Option( - False, "--no-cache", help="Execute the test even if results already exist", + False, + "--no-cache", + help="Execute the test even if results already exist", ), ): - """Associates a benchmark with a given mlcube or dataset. Only one option at a time. - """ + """Associates a benchmark with a given mlcube or dataset. Only one option at a time.""" AssociateBenchmark.run( benchmark_uid, model_uid, dataset_uid, approved=approval, no_cache=no_cache ) @@ -157,11 +161,12 @@ def run( help="Ignore failing model cubes, allowing for possibly submitting partial results", ), no_cache: bool = typer.Option( - False, "--no-cache", help="Execute even if results already exist", + False, + "--no-cache", + help="Execute even if results already exist", ), ): - """Runs the benchmark execution step for a given benchmark, prepared dataset and model - """ + """Runs the benchmark execution step for a given benchmark, prepared dataset and model""" BenchmarkExecution.run( benchmark_uid, data_uid, @@ -202,6 +207,5 @@ def view( help="Output file to store contents. If not provided, the output will be displayed", ), ): - """Displays the information of one or more benchmarks - """ + """Displays the information of one or more benchmarks""" EntityView.run(entity_id, Benchmark, format, local, mine, output) diff --git a/cli/medperf/commands/dataset/dataset.py b/cli/medperf/commands/dataset/dataset.py index 6e038560c..a7430add2 100644 --- a/cli/medperf/commands/dataset/dataset.py +++ b/cli/medperf/commands/dataset/dataset.py @@ -51,10 +51,9 @@ def create( ), name: str = typer.Option(..., "--name", help=NAME_HELP), description: str = typer.Option(..., "--description", help=DESC_HELP), - location: str = typer.Option(..., "--location", help=LOC_HELP) + location: str = typer.Option(..., "--location", help=LOC_HELP), ): - """Runs the Data preparation step for a specified benchmark and raw dataset - """ + """Runs the Data preparation step for a specified benchmark and raw dataset""" ui = config.ui data_uid = DataPreparation.run( benchmark_uid, @@ -79,8 +78,7 @@ def register( ), approval: bool = typer.Option(False, "-y", help="Skip approval step"), ): - """Submits an unregistered Dataset instance to the backend - """ + """Submits an unregistered Dataset instance to the backend""" ui = config.ui uid = DatasetRegistration.run(data_uid, approved=approval) ui.print("✅ Done!") @@ -96,7 +94,11 @@ def edit( name: str = typer.Option(None, "--name", help=NAME_HELP), description: str = typer.Option(None, "--description", help=DESC_HELP), location: str = typer.Option(None, "--location", help=LOC_HELP), - is_valid: bool = typer.Option(None, "--valid/--invalid", help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments") + is_valid: bool = typer.Option( + None, + "--valid/--invalid", + help="Flags a dataset valid/invalid. Invalid datasets can't be used for experiments", + ), ): """Edits a Dataset""" dset_info = { @@ -120,7 +122,9 @@ def associate( ), approval: bool = typer.Option(False, "-y", help="Skip approval step"), no_cache: bool = typer.Option( - False, "--no-cache", help="Execute the test even if results already exist", + False, + "--no-cache", + help="Execute the test even if results already exist", ), ): """Associate a registered dataset with a specific benchmark. @@ -159,6 +163,5 @@ def view( help="Output file to store contents. If not provided, the output will be displayed", ), ): - """Displays the information of one or more datasets - """ + """Displays the information of one or more datasets""" EntityView.run(entity_id, Dataset, format, local, mine, output) diff --git a/cli/medperf/commands/edit.py b/cli/medperf/commands/edit.py index fc4e7ebb2..1d5f1d00d 100644 --- a/cli/medperf/commands/edit.py +++ b/cli/medperf/commands/edit.py @@ -1,6 +1,7 @@ from medperf.entities.interface import Updatable from medperf.exceptions import InvalidEntityError + class EntityEdit: @staticmethod def run(entity_class, id: str, fields: dict): @@ -27,7 +28,7 @@ def prepare(self): def validate(self): if not isinstance(self.entity, Updatable): - raise InvalidEntityError("The passed entity can't be edited") + raise InvalidEntityError("The passed entity can't be edited") def edit(self): entity = self.entity @@ -36,4 +37,4 @@ def edit(self): if isinstance(entity, Updatable) and entity.is_registered: entity.update() - entity.write() \ No newline at end of file + entity.write() diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index f6d46437d..9a956bc83 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -233,8 +233,7 @@ def get_models_uids(cls, benchmark_uid: int) -> List[int]: return config.comms.get_benchmark_models(benchmark_uid) def edit(self, **kwargs): - """Edits a benchmark with the given property-value pairs - """ + """Edits a benchmark with the given property-value pairs""" data = self.todict() data.update(kwargs) new_bmk = Benchmark(**data) @@ -255,17 +254,18 @@ def __validate_edit(self, new_bmk: "Benchmark"): """ old_bmk = self # Field that shouldn't ber modified directly by the user - inmutable_fields = {"id",} + inmutable_fields = { + "id", + } # Fields that can no longer be modified while in production production_inmutable_fields = { "name", - "description" - "demo_dataset_tarball_hash", + "description" "demo_dataset_tarball_hash", "demo_dataset_generated_uid", "data_preparation_mlcube", "reference_model_mlcube", - "data_evaluator_mlcube" + "data_evaluator_mlcube", } if old_bmk.state == "OPERATION": @@ -278,14 +278,15 @@ def __validate_edit(self, new_bmk: "Benchmark"): if len(updated_inmutable_fields): fields_msg = ", ".join(updated_inmutable_fields) - msg = (f"The following fields can't be directly edited: "\ - + fields_msg \ - + ". For these changes, a new Benchmark is required") + msg = ( + "The following fields can't be directly edited: " + + fields_msg + + ". For these changes, a new Benchmark is required" + ) raise InvalidArgumentError(msg) def update(self): - """Updates the benchmark on the server - """ + """Updates the benchmark on the server""" if not self.is_registered: raise MedperfException("Can't update an unregistered benchmark") body = self.todict() diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index 9091dc829..7880b52bc 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -372,7 +372,7 @@ def __validate_edit(self, new_cube: "Cube"): if len(updated_inmutable_fields): fields_msg = ", ".join(updated_inmutable_fields) msg = ( - f"The following fields can't be directly edited: " + "The following fields can't be directly edited: " + fields_msg + ". For these changes, a new MLCube is required" ) diff --git a/cli/medperf/entities/dataset.py b/cli/medperf/entities/dataset.py index eaa782eb5..491d4af53 100644 --- a/cli/medperf/entities/dataset.py +++ b/cli/medperf/entities/dataset.py @@ -221,8 +221,7 @@ def upload(self): return updated_dataset_dict def edit(self, **kwargs): - """Edits a dataset with the given property-value pairs - """ + """Edits a dataset with the given property-value pairs""" data = self.todict() data.update(kwargs) new_dset = Dataset(**data) @@ -236,7 +235,7 @@ def __validate_edit(self, new_dset: "Dataset"): Args: new_dset (Dataset): The updated version of the same dataset - + Raises: InvalidArugmentError: The changed fields are not mutable """ @@ -252,12 +251,7 @@ def __validate_edit(self, new_dset: "Dataset"): } # Fields that can no longer be modified while in production - production_inmutable_fields = { - "name", - "split_seed", - "description", - "location" - } + production_inmutable_fields = {"name", "split_seed", "description", "location"} if old_dset.state == "OPERATION": inmutable_fields = inmutable_fields.union(production_inmutable_fields) @@ -268,14 +262,15 @@ def __validate_edit(self, new_dset: "Dataset"): if len(updated_inmutable_fields): fields_msg = ", ".join(updated_inmutable_fields) - msg = (f"The following fields can't be directly edited: " \ - + fields_msg \ - + ". For these changes, a new Dataset is required") + msg = ( + f"The following fields can't be directly edited: " + + fields_msg + + ". For these changes, a new Dataset is required" + ) raise InvalidArgumentError(msg) def update(self): - """Updates the benchmark on the server - """ + """Updates the benchmark on the server""" if not self.is_registered: raise MedperfException("Can't update an unregistered dataset") body = self.todict() diff --git a/cli/medperf/entities/interface.py b/cli/medperf/entities/interface.py index 5893f5a1e..95c716382 100644 --- a/cli/medperf/entities/interface.py +++ b/cli/medperf/entities/interface.py @@ -88,5 +88,4 @@ def edit(self, **kwargs): @abstractmethod def update(self): - """Updates the current entity on the server - """ \ No newline at end of file + """Updates the current entity on the server""" From a63ba697c79de2ad781e202d5cb5f4cac370a79d Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 31 May 2023 17:46:31 -0500 Subject: [PATCH 50/52] Fix linter issue --- cli/medperf/entities/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/entities/dataset.py b/cli/medperf/entities/dataset.py index 491d4af53..aa101e57e 100644 --- a/cli/medperf/entities/dataset.py +++ b/cli/medperf/entities/dataset.py @@ -263,7 +263,7 @@ def __validate_edit(self, new_dset: "Dataset"): if len(updated_inmutable_fields): fields_msg = ", ".join(updated_inmutable_fields) msg = ( - f"The following fields can't be directly edited: " + "The following fields can't be directly edited: " + fields_msg + ". For these changes, a new Dataset is required" ) From a1bed2c07d15848036f2c94b1ad6f760b41fd8b7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 31 May 2023 17:54:26 -0500 Subject: [PATCH 51/52] Fix tests --- cli/medperf/comms/entity_resources/resources.py | 2 +- cli/medperf/tests/entities/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/medperf/comms/entity_resources/resources.py b/cli/medperf/comms/entity_resources/resources.py index b88b674f0..b216d6eeb 100644 --- a/cli/medperf/comms/entity_resources/resources.py +++ b/cli/medperf/comms/entity_resources/resources.py @@ -117,7 +117,7 @@ def get_cube_image( def get_cube_additional( - url: str, cube_path: str, expected_tarball_hash: str = None, force: bool = True + url: str, cube_path: str, expected_tarball_hash: str = None, force: bool = False ) -> str: """Retrieves additional files of an MLCube. The additional files will be in a compressed tarball file. The function will additionally diff --git a/cli/medperf/tests/entities/utils.py b/cli/medperf/tests/entities/utils.py index bae15b4f7..f82df058e 100644 --- a/cli/medperf/tests/entities/utils.py +++ b/cli/medperf/tests/entities/utils.py @@ -85,7 +85,7 @@ def setup_cube_comms(mocker, comms, all_ents, user_ents, uploaded): def generate_cubefile_fn(fs, path, filename): # all_ids = [ent["id"] if type(ent) == dict else ent for ent in all_ents] - def cubefile_fn(url, cube_path, *args): + def cubefile_fn(url, cube_path, *args, **kwargs): if url == "broken_url": raise CommunicationRetrievalError filepath = os.path.join(cube_path, path, filename) From 56bb40178e4611c4ccfacb2baf836e2b1bebee15 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 2 Oct 2023 14:34:50 -0500 Subject: [PATCH 52/52] Set created entities to development by default --- cli/medperf/entities/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/entities/schemas.py b/cli/medperf/entities/schemas.py index 27abe0ee5..54de07085 100644 --- a/cli/medperf/entities/schemas.py +++ b/cli/medperf/entities/schemas.py @@ -91,7 +91,7 @@ def name_max_length(cls, v, *, values, **kwargs): class DeployableSchema(BaseModel): # TODO: This must change after allowing edits - state: str = "OPERATION" + state: str = "DEVELOPMENT" is_valid: bool = True