From 49b317f6ed34a48f331d3856165ed3bee2887d2e Mon Sep 17 00:00:00 2001 From: Tyler Sammons <233864410+b-tsammons@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:53:47 -1000 Subject: [PATCH 1/2] Change PySpark installation from Conda to pip pip handles installing required dependencies better without failing. I was able to get the docker running successfully using pip but kept hitting a resource allocation error with Conda even with 32 logical cores and 40gb of ram allocated to WSL 2 --- samples/.devcontainer/mariner/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/.devcontainer/mariner/Dockerfile b/samples/.devcontainer/mariner/Dockerfile index 7fa3cc9..47473a5 100644 --- a/samples/.devcontainer/mariner/Dockerfile +++ b/samples/.devcontainer/mariner/Dockerfile @@ -27,11 +27,11 @@ RUN tdnf install -y openssh-server nc \ # Install PySpark in the base Conda environment ENV SPARK_VERSION=3.4.1 -RUN conda install -c conda-forge pyspark==$SPARK_VERSION +RUN pip install pyspark==$SPARK_VERSION # Generate new host keys RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && \ chmod 600 /etc/ssh/ssh_host_rsa_key # Restart SSH daemon -CMD ["/bin/sh", "-c", "/usr/sbin/sshd -D"] \ No newline at end of file +CMD ["/bin/sh", "-c", "/usr/sbin/sshd -D"] From 91bf2ee96586b6340acfbd0f536fb1c981817f3f Mon Sep 17 00:00:00 2001 From: Tyler Sammons <233864410+b-tsammons@users.noreply.github.com> Date: Mon, 20 Oct 2025 20:19:44 -1000 Subject: [PATCH 2/2] Switch PySpark installation to conda with libmamba --- samples/.devcontainer/mariner/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/.devcontainer/mariner/Dockerfile b/samples/.devcontainer/mariner/Dockerfile index 47473a5..370c7bf 100644 --- a/samples/.devcontainer/mariner/Dockerfile +++ b/samples/.devcontainer/mariner/Dockerfile @@ -25,9 +25,11 @@ RUN tdnf install -y openssh-server nc \ && echo 'root:password' | chpasswd \ && sed -i 's/\#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# Install libmamba resolver +RUN conda install -n base conda-libmamba-solver # Install PySpark in the base Conda environment ENV SPARK_VERSION=3.4.1 -RUN pip install pyspark==$SPARK_VERSION +RUN conda install -c conda-forge pyspark==$SPARK_VERSION --solver=libmamba # Generate new host keys RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && \