update to spark v4.0.1 (#324)

Author: ihiverlet

.github/workflows/main-workflow.yml

@@ -83,7 +83,7 @@ jobs:
       base_image: python-minimal
       python_version_1: 3.13.8
       python_version_2: 3.12.12
-      spark_version: 3.5.7
+      spark_version: 4.0.1
       build_gpu: false
     secrets: inherit
   jupyter-pyspark:
@@ -95,7 +95,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.13.8
       python_version_2: 3.12.12
-      spark_version: 3.5.7
+      spark_version: 4.0.1
       build_gpu: false
     secrets: inherit
   vscode-pyspark:
@@ -107,7 +107,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.13.8
       python_version_2: 3.12.12
-      spark_version: 3.5.7
+      spark_version: 4.0.1
       build_gpu: false
     secrets: inherit
   python-pytorch:
@@ -189,7 +189,7 @@ jobs:
       base_image: r-minimal
       r_version_1: 4.5.2
       r_version_2: 4.4.3
-      spark_version: 3.5.7
+      spark_version: 4.0.1
       build_gpu: false
     secrets: inherit
   rstudio-sparkr:
@@ -201,7 +201,7 @@ jobs:
       base_image: sparkr
       r_version_1: 4.5.2
       r_version_2: 4.4.3
-      spark_version: 3.5.7
+      spark_version: 4.0.1
       build_gpu: false
     secrets: inherit
   r-python-julia: