diff --git a/src/main/scala/ignition/core/jobs/CoreJobRunner.scala b/src/main/scala/ignition/core/jobs/CoreJobRunner.scala index 2c904b5c..5d8b6e04 100644 --- a/src/main/scala/ignition/core/jobs/CoreJobRunner.scala +++ b/src/main/scala/ignition/core/jobs/CoreJobRunner.scala @@ -80,6 +80,10 @@ object CoreJobRunner { val builder = SparkSession.builder + builder.config("spark.executor.memory", config.executorMemory) + builder.config("spark.eventLog.dir", "file:///media/tmp/spark-events") + + builder.master(config.master) builder.appName(appName) builder.config("spark.hadoop.mapred.output.committer.class", classOf[DirectOutputCommitter].getName()) diff --git a/tools/cluster.py b/tools/cluster.py index 966dd987..421b6940 100755 --- a/tools/cluster.py +++ b/tools/cluster.py @@ -245,8 +245,6 @@ def launch(cluster_name, slaves, assert not master_instance_type or master_instance_type == instance_type, 'Different master instance type is currently unsupported' assert not master_ami or master_ami == ami, 'Different master ami is currently unsupported' - assert not ondemand, 'On demand is unsupported' - assert master_spot, 'On demand master is currently unsupported' all_args = locals()