fix(cluster): resolve DNS failures on systemd-resolved hosts

Docker's embedded DNS at 127.0.0.11 is only reachable from the
container's own network namespace. k3s pods in child namespaces
cannot reach it, causing silent DNS failures on Ubuntu and other
systemd-resolved hosts where /etc/resolv.conf contains 127.0.0.53.

Sniff upstream DNS resolvers from the host in the Rust bootstrap
crate by reading /run/systemd/resolve/resolv.conf (systemd-resolved
only — intentionally does NOT read /etc/resolv.conf to avoid
bypassing Docker Desktop's DNAT proxy on macOS/Windows). Filter
loopback addresses (127.x.x.x and ::1) and pass the result to
the container as the UPSTREAM_DNS env var. Skip DNS sniffing for
remote deploys where the local host's resolvers would be wrong.

The entrypoint checks UPSTREAM_DNS first, falling back to
/etc/resolv.conf inside the container for manual launches. This
follows the existing pattern used by registry config, SSH gateway,
GPU support, and image tags.

Closes #437

Signed-off-by: Brian Taylor <brian.taylor818@gmail.com>

Author: brianwtaylor

crates/openshell-bootstrap/src/docker.rs

@@ -236,6 +236,57 @@ fn home_dir() -> Option<String> {
     std::env::var("HOME").ok()
 }
 
+/// Discover upstream DNS resolvers from systemd-resolved's configuration.
+///
+/// Only reads `/run/systemd/resolve/resolv.conf` — the upstream resolver file
+/// maintained by systemd-resolved. This file is only present on Linux hosts
+/// running systemd-resolved (e.g., Ubuntu), so the function is a no-op on
+/// macOS, Windows/WSL, and non-systemd Linux distributions.
+///
+/// We intentionally do NOT fall back to `/etc/resolv.conf` here. On Docker
+/// Desktop (macOS/Windows), `/etc/resolv.conf` may contain non-loopback
+/// resolvers that appear valid but are unreachable via direct UDP from inside
+/// the container's network stack. Those environments rely on the entrypoint's
+/// iptables DNAT proxy to Docker's embedded DNS — sniffing host resolvers
+/// would bypass that proxy and break DNS.
+///
+/// Returns an empty vec if no usable resolvers are found.
+fn resolve_upstream_dns() -> Vec<String> {
+    let paths = ["/run/systemd/resolve/resolv.conf"];
+
+    for path in &paths {
+        if let Ok(contents) = std::fs::read_to_string(path) {
+            let resolvers: Vec<String> = contents
+                .lines()
+                .filter_map(|line| {
+                    let line = line.trim();
+                    if !line.starts_with("nameserver") {
+                        return None;
+                    }
+                    let ip = line.split_whitespace().nth(1)?;
+                    if ip.starts_with("127.") || ip == "::1" {
+                        return None;
+                    }
+                    Some(ip.to_string())
+                })
+                .collect();
+
+            if !resolvers.is_empty() {
+                tracing::debug!(
+                    "Discovered {} upstream DNS resolver(s) from {}: {}",
+                    resolvers.len(),
+                    path,
+                    resolvers.join(", "),
+                );
+                return resolvers;
+            }
+        }
+    }
+
+    tracing::debug!("No upstream DNS resolvers found in host resolver config");
+    Vec::new()
+}
+
 /// Create an SSH Docker client from remote options.
 pub async fn create_ssh_docker_client(remote: &RemoteOptions) -> Result<Docker> {
     // Ensure destination has ssh:// prefix
@@ -455,6 +506,7 @@ pub async fn ensure_container(
     registry_username: Option<&str>,
     registry_token: Option<&str>,
     gpu: bool,
+    is_remote: bool,
 ) -> Result<()> {
     let container_name = container_name(name);
 
@@ -675,6 +727,17 @@ pub async fn ensure_container(
         env_vars.push("GPU_ENABLED=true".to_string());
     }
 
+    // Pass upstream DNS resolvers discovered on the host so the entrypoint
+    // can configure k3s without probing files inside the container.
+    // Skip for remote deploys — the local host's resolvers are likely wrong
+    // for the remote Docker host (different network, split-horizon DNS, etc.).
+    if !is_remote {
+        let upstream_dns = resolve_upstream_dns();
+        if !upstream_dns.is_empty() {
+            env_vars.push(format!("UPSTREAM_DNS={}", upstream_dns.join(",")));
+        }
+    }
+
     let env = Some(env_vars);
 
     let config = ContainerCreateBody {
@@ -1195,4 +1258,112 @@ mod tests {
             "should return a reasonable number of sockets"
         );
     }
+
+    #[test]
+    fn resolve_upstream_dns_filters_loopback() {
+        // This test validates the function runs without panic on the current host.
+        // The exact output depends on the host's DNS config, but loopback
+        // addresses must never appear in the result.
+        let resolvers = resolve_upstream_dns();
+        for r in &resolvers {
+            assert!(
+                !r.starts_with("127."),
+                "IPv4 loopback should be filtered: {r}"
+            );
+            assert_ne!(r, "::1", "IPv6 loopback should be filtered");
+        }
+    }
+
+    #[test]
+    fn resolve_upstream_dns_returns_vec() {
+        // Verify the function returns a vec (may be empty in some CI environments
+        // where no resolv.conf exists, but should never panic).
+        let resolvers = resolve_upstream_dns();
+        assert!(
+            resolvers.len() <= 20,
+            "should return a reasonable number of resolvers"
+        );
+    }
+
+    /// Helper: parse resolv.conf content using the same logic as resolve_upstream_dns().
+    /// Allows deterministic testing without depending on host DNS config.
+    fn parse_resolv_conf(contents: &str) -> Vec<String> {
+        contents
+            .lines()
+            .filter_map(|line| {
+                let line = line.trim();
+                if !line.starts_with("nameserver") {
+                    return None;
+                }
+                let ip = line.split_whitespace().nth(1)?;
+                if ip.starts_with("127.") || ip == "::1" {
+                    return None;
+                }
+                Some(ip.to_string())
+            })
+            .collect()
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv4_loopback() {
+        let input = "nameserver 127.0.0.1\nnameserver 127.0.0.53\nnameserver 127.0.0.11\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv6_loopback() {
+        let input = "nameserver ::1\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_passes_real_resolvers() {
+        let input = "nameserver 8.8.8.8\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_mixed_loopback_and_real() {
+        let input =
+            "nameserver 127.0.0.53\nnameserver ::1\nnameserver 10.0.0.1\nnameserver 172.16.0.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["10.0.0.1", "172.16.0.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_ignores_comments_and_other_lines() {
+        let input =
+            "# nameserver 8.8.8.8\nsearch example.com\noptions ndots:5\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_handles_tabs_and_extra_spaces() {
+        let input = "nameserver\t8.8.8.8\nnameserver     1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_empty_input() {
+        assert!(parse_resolv_conf("").is_empty());
+        assert!(parse_resolv_conf("   \n\n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_bare_nameserver_keyword() {
+        assert!(parse_resolv_conf("nameserver\n").is_empty());
+        assert!(parse_resolv_conf("nameserver   \n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_systemd_resolved_typical() {
+        let input =
+            "# This is /run/systemd/resolve/resolv.conf\nnameserver 192.168.1.1\nsearch lan\n";
+        assert_eq!(parse_resolv_conf(input), vec!["192.168.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_crlf_line_endings() {
+        let input = "nameserver 8.8.8.8\r\nnameserver 1.1.1.1\r\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
 }

crates/openshell-bootstrap/src/lib.rs

@@ -417,6 +417,7 @@ where
             registry_username.as_deref(),
             registry_token.as_deref(),
             gpu,
+            remote_opts.is_some(),
         )
         .await?;
         start_container(&target_docker, &name).await?;

deploy/docker/cluster-entrypoint.sh

@@ -69,7 +69,46 @@ wait_for_default_route() {
 #   3. Adding DNAT rules so traffic to <eth0_ip>:53 reaches Docker's DNS
 #   4. Writing that IP into the k3s resolv.conf
 
+# Extract upstream DNS resolvers reachable from k3s pod namespaces.
+# Docker's embedded DNS (127.0.0.11) is namespace-local — DNAT to it from
+# pod traffic is dropped as a martian packet. Use real upstream servers instead.
+#
+# Priority:
+#   1. UPSTREAM_DNS env var (set by bootstrap, comma-separated)
+#   2. /etc/resolv.conf (fallback for non-bootstrap launches)
+get_upstream_resolvers() {
+    local resolvers=""
+
+    # Bootstrap-provided resolvers (sniffed from host by the Rust bootstrap crate)
+    if [ -n "${UPSTREAM_DNS:-}" ]; then
+        resolvers=$(printf '%s\n' "$UPSTREAM_DNS" | tr ',' '\n' | \
+            awk '{ip=$1; if(ip !~ /^127\./ && ip != "::1" && ip != "") print ip}')
+    fi
+
+    # Fallback: Docker-generated resolv.conf may have non-loopback servers
+    if [ -z "$resolvers" ]; then
+        resolvers=$(awk '/^nameserver/{ip=$2; gsub(/\r/,"",ip); if(ip !~ /^127\./ && ip != "::1") print ip}' \
+            /etc/resolv.conf)
+    fi
+
+    echo "$resolvers"
+}
+
 setup_dns_proxy() {
+    # Prefer upstream resolvers that work across network namespaces.
+    # This avoids the DNAT-to-loopback problem on systemd-resolved hosts.
+    UPSTREAM_DNS=$(get_upstream_resolvers)
+    if [ -n "$UPSTREAM_DNS" ]; then
+        : > "$RESOLV_CONF"
+        echo "$UPSTREAM_DNS" | while read -r ns; do
+            [ -n "$ns" ] && echo "nameserver $ns" >> "$RESOLV_CONF"
+        done
+        echo "DNS: using upstream resolvers directly (avoids cross-namespace DNAT)"
+        cat "$RESOLV_CONF"
+        return 0
+    fi
+
+    # Fall back to DNAT proxy when no upstream resolvers are available.
     # Extract Docker's actual DNS listener ports from the DOCKER_OUTPUT chain.
     # Docker sets up rules like:
     #   -A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp --dport 53 -j DNAT --to-destination 127.0.0.11:<port>
@@ -160,6 +199,8 @@ verify_dns() {
         sleep 1
         i=$((i + 1))
     done
+    echo "Warning: DNS verification failed for $lookup_host after $attempts attempts"
+    echo "  resolv.conf: $(head -3 "$RESOLV_CONF" 2>/dev/null)"
     return 1
 }