Merge branch 'master' of github.com:Paperspace/paperspace-python into v0.2.0a2

Author: BartoszCki

paperspace/cli/__init__.py

@@ -5,6 +5,7 @@
 import paperspace.cli.auth
 import paperspace.cli.deployments
 import paperspace.cli.experiments
+import paperspace.cli.hyperparameters
 import paperspace.cli.jobs
 import paperspace.cli.machines
 import paperspace.cli.models

paperspace/cli/deployments.py

@@ -100,7 +100,7 @@ def create_deployment(api_key=None, **kwargs):
 @click.option(
     "--modelId",
     "modelId",
-    help="Use to filter by project ID",
+    help="Use to filter by model ID",
 )
 @api_key_option
 def get_deployments_list(api_key=None, **filters):

paperspace/cli/experiments.py

@@ -58,6 +58,11 @@ def common_experiments_create_options(f):
             "workspaceUrl",
             help="Project git repository url",
         ),
+        click.option(
+            "--ignoreFiles",
+            "ignore_files",
+            help="Ignore certain files from uploading"
+        ),
         click.option(
             "--workingDirectory",
             "workingDirectory",

paperspace/cli/hyperparameters.py

@@ -0,0 +1,136 @@
+import functools
+
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import ClickGroup
+from paperspace.commands import hyperparameters as hyperparameters_commands
+
+
+@cli.group("hyperparameters", help="Manage hyperparameters", cls=ClickGroup)
+def hyperparameters_group():
+    pass
+
+
+def common_hyperparameter_create_options(f):
+    options = [
+        click.option(
+            "--name",
+            "name",
+            required=True,
+        ),
+        click.option(
+            "--projectId",
+            "projectHandle",
+            required=True,
+        ),
+        click.option(
+            "--tuningCommand",
+            "tuningCommand",
+            required=True,
+        ),
+        click.option(
+            "--workerContainer",
+            "workerContainer",
+            required=True,
+        ),
+        click.option(
+            "--workerMachineType",
+            "workerMachineType",
+            required=True,
+        ),
+        click.option(
+            "--workerCommand",
+            "workerCommand",
+            required=True,
+        ),
+        click.option(
+            "--workerCount",
+            "workerCount",
+            required=True,
+            type=int,
+        ),
+        click.option(
+            "--serverRegistryUsername",
+            "hyperparameterServerRegistryUsername",
+        ),
+        click.option(
+            "--serverRegistryPassword",
+            "hyperparameterServerRegistryPassword",
+        ),
+        click.option(
+            "--serverContainerUser",
+            "hyperparameterServerContainerUser",
+        ),
+    ]
+    return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
+
+
+@hyperparameters_group.command("create", help="Create hyperparameter")
+@common_hyperparameter_create_options
+@common.api_key_option
+def create_hyperparameter(api_key, **hyperparameter):
+    common.del_if_value_is_none(hyperparameter)
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.CreateHyperparameterCommand(api=hyperparameters_api)
+    command.execute(hyperparameter)
+
+
+@hyperparameters_group.command("createAndStart", help="Create hyperparameter")
+@common_hyperparameter_create_options
+@common.api_key_option
+def create_and_start_hyperparameter(api_key, **hyperparameter):
+    common.del_if_value_is_none(hyperparameter)
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.CreateAndStartHyperparameterCommand(api=hyperparameters_api)
+    command.execute(hyperparameter)
+
+
+@hyperparameters_group.command("list", help="List hyperparameters")
+@common.api_key_option
+def list_hyperparameters(api_key):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.ListHyperparametersCommand(api=hyperparameters_api)
+    command.execute()
+
+
+# TODO: 'unhidden' command and test it when api is updated to support deleting hyperparameters
+@hyperparameters_group.command("delete", help="Delete hyperparameter", hidden=True)
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def delete_hyperparameter(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.DeleteHyperparameterCommand(api=hyperparameters_api)
+    command.execute(id_)
+
+
+@hyperparameters_group.command("details", help="Show details of hyperparameter")
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def get_hyperparameter_details(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.HyperparameterDetailsCommand(api=hyperparameters_api)
+    command.execute(id_)
+
+
+@hyperparameters_group.command("start", help="Start hyperparameter tuning")
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def start_hyperparameter_tuning(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.HyperparameterStartCommand(api=hyperparameters_api)
+    command.execute(id_)

paperspace/cli/jobs.py

@@ -1,10 +1,9 @@
 import click
 
 from paperspace import client, config
-from paperspace.cli import common
-from paperspace.cli.cli_types import json_string
-from paperspace.cli.common import del_if_value_is_none, ClickGroup
 from paperspace.cli.cli import cli
+from paperspace.cli.cli_types import json_string
+from paperspace.cli.common import api_key_option, del_if_value_is_none, ClickGroup
 from paperspace.commands import jobs as jobs_commands
 
 
@@ -20,7 +19,7 @@ def jobs_group():
     required=True,
     help="Delete job with given ID",
 )
-@common.api_key_option
+@api_key_option
 def delete_job(job_id, api_key=None):
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = jobs_commands.DeleteJobCommand(api=jobs_api)
@@ -34,7 +33,7 @@ def delete_job(job_id, api_key=None):
     required=True,
     help="Stop job with given ID",
 )
-@common.api_key_option
+@api_key_option
 def stop_job(job_id, api_key=None):
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = jobs_commands.StopJobCommand(api=jobs_api)
@@ -57,9 +56,9 @@ def stop_job(job_id, api_key=None):
     "experimentId",
     help="Use to filter jobs by experiment ID",
 )
-@common.api_key_option
+@api_key_option
 def list_jobs(api_key, **filters):
-    common.del_if_value_is_none(filters)
+    del_if_value_is_none(filters)
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = jobs_commands.ListJobsCommand(api=jobs_api)
     command.execute(filters=filters)
@@ -75,7 +74,8 @@ def list_jobs(api_key, **filters):
 @click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
 @click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
 @click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
-@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", )
+@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
+@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
 @click.option("--experimentId", "experimentId", help="Experiment Id")
 @click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
 @click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
@@ -86,7 +86,7 @@ def list_jobs(api_key, **filters):
 @click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
 @click.option("--registryUsername", "registryUsername", help="Docker registry username")
 @click.option("--registryPassword", "registryPassword", help="Docker registry password")
-@common.api_key_option
+@api_key_option
 def create_job(api_key, **kwargs):
     del_if_value_is_none(kwargs)
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
@@ -100,8 +100,44 @@ def create_job(api_key, **kwargs):
     "job_id",
     required=True
 )
-@common.api_key_option
+@api_key_option
 def list_logs(job_id, api_key=None):
     logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
     command = jobs_commands.JobLogsCommand(api=logs_api)
     command.execute(job_id)
+
+
+@jobs_group.group("artifacts", help="Manage jobs' artifacts", cls=ClickGroup)
+def artifacts():
+    pass
+
+
+@artifacts.command("destroy", help="Destroy job's artifacts")
+@click.argument("job_id")
+@click.option("--files", "files")
+@api_key_option
+def destroy_artifacts(job_id, api_key=None, files=None):
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = jobs_commands.ArtifactsDestroyCommand(api=jobs_api)
+    command.execute(job_id, files=files)
+
+
+@artifacts.command("get", help="Get job's artifacts")
+@click.argument("job_id")
+@api_key_option
+def get_artifacts(job_id, api_key=None):
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = jobs_commands.ArtifactsGetCommand(api=jobs_api)
+    command.execute(job_id)
+
+
+@artifacts.command("list", help="List job's artifacts")
+@click.argument("job_id")
+@click.option("--size", "-s", "size", help="Show file size", is_flag=True)
+@click.option("--links", "-l", "links", help="Show file URL", is_flag=True)
+@click.option("--files", "files", help="Get only given file (use at the end * as a wildcard)")
+@api_key_option
+def list_artifacts(job_id, size, links, files, api_key=None):
+    jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+    command = jobs_commands.ArtifactsListCommand(api=jobs_api)
+    command.execute(job_id=job_id, size=size, links=links, files=files)

paperspace/client.py

@@ -30,26 +30,35 @@ def get_path(self, url):
 
     def post(self, url, json=None, params=None, files=None):
         path = self.get_path(url)
-        response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
         logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
-                     .format(response.url, self.headers, json, params))
+                     .format(path, self.headers, json, params))
+        response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))
         return response
 
     def put(self, url, json=None, params=None):
         path = self.get_path(url)
-        response = requests.put(path, json=json, params=params, headers=self.headers)
         logger.debug("PUT request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
-                     .format(response.url, self.headers, json, params))
+                     .format(path, self.headers, json, params))
+        response = requests.put(path, json=json, params=params, headers=self.headers)
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))
         return response
 
     def get(self, url, json=None, params=None):
         path = self.get_path(url)
-        response = requests.get(path, params=params, headers=self.headers, json=json)
         logger.debug("GET request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
+                     .format(path, self.headers, json, params))
+        response = requests.get(path, params=params, headers=self.headers, json=json)
+        logger.debug("Response status code: {}".format(response.status_code))
+        logger.debug("Response content: {}".format(response.content))
+        return response
+
+    def delete(self, url, json=None, params=None):
+        path = self.get_path(url)
+        response = requests.delete(path, params=params, headers=self.headers, json=json)
+        logger.debug("DELETE request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
                      .format(response.url, self.headers, json, params))
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))

paperspace/commands/__init__.py

@@ -0,0 +1,17 @@
+from collections import OrderedDict
+
+from paperspace import logger
+
+
+class CommandBase(object):
+    def __init__(self, api=None, logger_=logger):
+        self.api = api
+        self.logger = logger_
+
+    def _print_dict_recursive(self, input_dict, indent=0, tabulator="  "):
+        for key, val in input_dict.items():
+            self.logger.log("%s%s:" % (tabulator * indent, key))
+            if type(val) is dict:
+                self._print_dict_recursive(OrderedDict(val), indent + 1)
+            else:
+                self.logger.log("%s%s" % (tabulator * (indent + 1), val))