From ca8776a58680579d8710245badf1aeeefeefea05 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 14 Oct 2025 19:51:51 -0700 Subject: [PATCH] [SPARK-53912] Add `cluster-preview.yaml` and `spark-connect-server-preview.yaml` --- examples/cluster-preview.yaml | 32 ++++++++++++++++++++++ examples/spark-connect-server-preview.yaml | 31 +++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 examples/cluster-preview.yaml create mode 100644 examples/spark-connect-server-preview.yaml diff --git a/examples/cluster-preview.yaml b/examples/cluster-preview.yaml new file mode 100644 index 00000000..7e94de56 --- /dev/null +++ b/examples/cluster-preview.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: spark.apache.org/v1 +kind: SparkCluster +metadata: + name: cluster-preview +spec: + runtimeVersions: + sparkVersion: "4.1.0-preview2" + clusterTolerations: + instanceConfig: + initWorkers: 3 + minWorkers: 3 + maxWorkers: 3 + sparkConf: + spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}" + spark.master.ui.title: "Prod Spark Cluster" + spark.master.rest.enabled: "true" + spark.master.rest.host: "0.0.0.0" + spark.ui.reverseProxy: "true" diff --git a/examples/spark-connect-server-preview.yaml b/examples/spark-connect-server-preview.yaml new file mode 100644 index 00000000..d4cb9342 --- /dev/null +++ b/examples/spark-connect-server-preview.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: spark.apache.org/v1 +kind: SparkApplication +metadata: + name: spark-connect-server-preview +spec: + mainClass: "org.apache.spark.sql.connect.service.SparkConnectServer" + sparkConf: + spark.dynamicAllocation.enabled: "true" + spark.dynamicAllocation.shuffleTracking.enabled: "true" + spark.dynamicAllocation.minExecutors: "3" + spark.dynamicAllocation.maxExecutors: "3" + spark.kubernetes.authenticate.driver.serviceAccountName: "spark" + spark.kubernetes.container.image: "apache/spark:{{SPARK_VERSION}}" + applicationTolerations: + resourceRetainPolicy: OnFailure + runtimeVersions: + sparkVersion: "4.1.0-preview2"