From 5b73250445bea8ac34266eceb272b660ef7b75aa Mon Sep 17 00:00:00 2001 From: Daniel Pereira Date: Fri, 4 Mar 2022 09:40:28 -0400 Subject: [PATCH 1/2] Use set spark.executor.memory in flintrock --- src/main/scala/ignition/core/jobs/CoreJobRunner.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/scala/ignition/core/jobs/CoreJobRunner.scala b/src/main/scala/ignition/core/jobs/CoreJobRunner.scala index 2c904b5..5d8b6e0 100644 --- a/src/main/scala/ignition/core/jobs/CoreJobRunner.scala +++ b/src/main/scala/ignition/core/jobs/CoreJobRunner.scala @@ -80,6 +80,10 @@ object CoreJobRunner { val builder = SparkSession.builder + builder.config("spark.executor.memory", config.executorMemory) + builder.config("spark.eventLog.dir", "file:///media/tmp/spark-events") + + builder.master(config.master) builder.appName(appName) builder.config("spark.hadoop.mapred.output.committer.class", classOf[DirectOutputCommitter].getName()) From e0be3ae0fa1322cbe0f2d14928c218409e32ad38 Mon Sep 17 00:00:00 2001 From: Daniel Pereira Date: Tue, 15 Mar 2022 10:18:00 -0400 Subject: [PATCH 2/2] Support execution of ondemand instances --- tools/cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/cluster.py b/tools/cluster.py index 966dd98..421b694 100755 --- a/tools/cluster.py +++ b/tools/cluster.py @@ -245,8 +245,6 @@ def launch(cluster_name, slaves, assert not master_instance_type or master_instance_type == instance_type, 'Different master instance type is currently unsupported' assert not master_ami or master_ami == ami, 'Different master ami is currently unsupported' - assert not ondemand, 'On demand is unsupported' - assert master_spot, 'On demand master is currently unsupported' all_args = locals()