PLAT-144996: Fixing spark ui base path url

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -500,8 +500,22 @@ class SparkContext(config: SparkConf) extends Logging {
 
     _ui =
       if (conf.get(UI_ENABLED)) {
-        Some(SparkUI.create(Some(this), _statusStore, _conf, _env.securityManager, appName, "",
-          startTime))
+        // Prefer spark.ui.proxyBasePath; fall back to spark.ui.proxyBase for backward
+        // compatibility with infrastructure that passes --conf spark.ui.proxyBase=... directly.
+        val rawBasePath = _conf.get(UI_PROXY_BASE_PATH)
+          .orElse(_conf.getOption("spark.ui.proxyBase"))
+          .getOrElse("")
+        val basePath = if (rawBasePath.nonEmpty) {
+          val normalized = "/" + rawBasePath.stripPrefix("/").stripSuffix("/")
+          logInfo(s"Spark UI proxyBasePath configured: " +
+            s"raw='$rawBasePath', normalized='$normalized'")
+          System.setProperty("spark.ui.proxyBase", normalized)
+          normalized
+        } else {
+          ""
+        }
+        Some(SparkUI.create(Some(this), _statusStore, _conf, _env.securityManager, appName,
+          basePath, startTime))
       } else {
         // For tests, do not enable the UI
         None
@@ -617,7 +631,12 @@ class SparkContext(config: SparkConf) extends Logging {
       _conf.set(ShuffleDataIOUtils.SHUFFLE_SPARK_CONF_PREFIX + k, v)
     }
 
-    if (_conf.get(UI_REVERSE_PROXY)) {
+    if (_conf.get(UI_REVERSE_PROXY) && _conf.get(UI_PROXY_BASE_PATH).isEmpty &&
+        _conf.getOption("spark.ui.proxyBase").isEmpty) {
+      // Only apply YARN-style reverse proxy URL when no explicit proxyBasePath is configured.
+      // If spark.ui.proxyBasePath (or spark.ui.proxyBase) is set, SparkUI.create already
+      // mounted handlers at the correct prefixed paths and set spark.ui.proxyBase --
+      // clobbering it here would break link generation in all generated HTML.
       val proxyUrl = _conf.get(UI_REVERSE_PROXY_URL).getOrElse("").stripSuffix("/")
       System.setProperty("spark.ui.proxyBase", proxyUrl + "/proxy/" + _applicationId)
     }

core/src/main/scala/org/apache/spark/internal/config/UI.scala

@@ -214,6 +214,14 @@ private[spark] object UI {
     .stringConf
     .createOptional
 
+  val UI_PROXY_BASE_PATH = ConfigBuilder("spark.ui.proxyBasePath")
+    .doc("Base path prefix for the Spark UI. Used when Spark is served behind a reverse proxy " +
+      "at a non-root path. All UI servlet handlers will be mounted under this prefix, and " +
+      "spark.ui.proxyBase system property will be set so UIUtils.uiRoot() generates correct links.")
+    .version("3.5.0")
+    .stringConf
+    .createOptional
+
   val CUSTOM_EXECUTOR_LOG_URL = ConfigBuilder("spark.ui.custom.executor.log.url")
     .doc("Specifies custom spark executor log url for supporting external log service instead of " +
       "using cluster managers' application log urls in the Spark UI. Spark will support " +

core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala

@@ -28,7 +28,7 @@ import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 
 import org.apache.spark.SecurityManager
-import org.apache.spark.ui.{SparkUI, UIUtils}
+import org.apache.spark.ui.{JettyUtils, SparkUI, UIUtils}
 
 /**
  * Main entry point for serving spark application metrics as json, using JAX-RS.
@@ -57,9 +57,17 @@ private[v1] class ApiRootResource extends ApiRequestContext {
 
 private[spark] object ApiRootResource {
 
-  def getServletHandler(uiRoot: UIRoot): ServletContextHandler = {
+  def getServletHandler(uiRoot: UIRoot, basePath: String = ""): ServletContextHandler = {
     val jerseyContext = new ServletContextHandler(ServletContextHandler.NO_SESSIONS)
-    jerseyContext.setContextPath("/api")
+    val contextPath = if (basePath.nonEmpty) {
+      (basePath + "/api").stripSuffix("/")
+    } else {
+      "/api"
+    }
+    jerseyContext.setContextPath(contextPath)
+    if (basePath.nonEmpty) {
+      jerseyContext.setAttribute(JettyUtils.PROXY_BASE_PATH_ATTRIBUTE, basePath)
+    }
     val holder: ServletHolder = new ServletHolder(classOf[ServletContainer])
     holder.setInitParameter(ServerProperties.PROVIDER_PACKAGES, "org.apache.spark.status.api.v1")
     UIRootFromServletContext.setUiRoot(jerseyContext, uiRoot)

core/src/main/scala/org/apache/spark/ui/JettyUtils.scala

@@ -17,7 +17,7 @@
 
 package org.apache.spark.ui
 
-import java.net.{URI, URL, URLDecoder}
+import java.net.{URI, URLDecoder}
 import java.util.EnumSet
 import javax.servlet.DispatcherType
 import javax.servlet.http._
@@ -104,6 +104,8 @@ private[spark] object JettyUtils extends Logging {
     createServletHandler(path, createServlet(servletParams, conf), basePath)
   }
 
+  val PROXY_BASE_PATH_ATTRIBUTE = "spark.ui.proxyBasePath"
+
   /** Create a context handler that responds to a request with the given path prefix */
   def createServletHandler(
       path: String,
@@ -117,6 +119,9 @@ private[spark] object JettyUtils extends Logging {
     val contextHandler = new ServletContextHandler
     val holder = new ServletHolder(servlet)
     contextHandler.setContextPath(prefixedPath)
+    if (basePath.nonEmpty) {
+      contextHandler.setAttribute(PROXY_BASE_PATH_ATTRIBUTE, basePath)
+    }
     contextHandler.addServlet(holder, "/")
     contextHandler
   }
@@ -128,7 +133,6 @@ private[spark] object JettyUtils extends Logging {
       beforeRedirect: HttpServletRequest => Unit = x => (),
       basePath: String = "",
       httpMethods: Set[String] = Set("GET")): ServletContextHandler = {
-    val prefixedDestPath = basePath + destPath
     val servlet = new HttpServlet {
       override def doGet(request: HttpServletRequest, response: HttpServletResponse): Unit = {
         if (httpMethods.contains("GET")) {
@@ -146,9 +150,29 @@ private[spark] object JettyUtils extends Logging {
       }
       private def doRequest(request: HttpServletRequest, response: HttpServletResponse): Unit = {
         beforeRedirect(request)
-        // Make sure we don't end up with "//" in the middle
-        val newUrl = new URL(new URL(request.getRequestURL.toString), prefixedDestPath).toString
-        response.sendRedirect(newUrl)
+        // Derive the basePath dynamically from the matched context path at request time.
+        // request.getContextPath() is set by Jetty to the handler's mounted context path,
+        // which already includes the basePath (e.g. "/sparkrb/.../sparkui").
+        // Stripping srcPath from it gives us the effective basePath, regardless of whether
+        // spark.ui.proxyBasePath was configured.
+        val contextPath = request.getContextPath
+        val effectiveBasePath = if (srcPath == "/") {
+          contextPath
+        } else {
+          contextPath.stripSuffix(srcPath)
+        }
+        val prefixedDestPath = (effectiveBasePath + destPath).replaceAll("//+", "/")
+        // Set Location header directly instead of calling response.sendRedirect().
+        // sendRedirect() in the Servlet spec always converts the path to an absolute URL using
+        // request.getServerName()/getServerPort() — the internal Spark address, not the external
+        // proxy URL. This causes the browser to receive e.g. http://10.x.x.x:4040/myapp/jobs/
+        // which it cannot reach. By setting the header ourselves we send the raw path and let
+        // the browser resolve it against the external proxy origin it actually used.
+        // RFC 7231 allows relative references in the Location header and all modern browsers
+        // handle them correctly.
+        response.setStatus(HttpServletResponse.SC_FOUND)
+        response.setHeader("Location", prefixedDestPath)
+        logDebug(s"Redirect: ${request.getRequestURI} -> $prefixedDestPath")
       }
       // SPARK-5983 ensure TRACE is not supported
       protected override def doTrace(req: HttpServletRequest, res: HttpServletResponse): Unit = {
@@ -159,7 +183,10 @@ private[spark] object JettyUtils extends Logging {
   }
 
   /** Create a handler for serving files from a static directory */
-  def createStaticHandler(resourceBase: String, path: String): ServletContextHandler = {
+  def createStaticHandler(
+      resourceBase: String,
+      path: String,
+      basePath: String = ""): ServletContextHandler = {
     val contextHandler = new ServletContextHandler
     contextHandler.setInitParameter("org.eclipse.jetty.servlet.Default.gzip", "false")
     val staticHandler = new DefaultServlet
@@ -170,7 +197,15 @@ private[spark] object JettyUtils extends Logging {
       case None =>
         throw new Exception("Could not find resource path for Web UI: " + resourceBase)
     }
-    contextHandler.setContextPath(path)
+    val prefixedPath = if (basePath.nonEmpty) {
+      (basePath + path).stripSuffix("/")
+    } else {
+      path
+    }
+    contextHandler.setContextPath(prefixedPath)
+    if (basePath.nonEmpty) {
+      contextHandler.setAttribute(PROXY_BASE_PATH_ATTRIBUTE, basePath)
+    }
     contextHandler.addServlet(holder, "/")
     contextHandler
   }
@@ -570,7 +605,7 @@ private[spark] case class ServerInfo(
  * a servlet context without the trailing slash (e.g. "/jobs") - Jetty will send a redirect to the
  * same URL, but with a trailing slash.
  */
-private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper {
+private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper with Logging {
 
   private val proxyUri = _proxyUri.stripSuffix("/")
 
@@ -590,14 +625,36 @@ private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper {
     override def sendRedirect(location: String): Unit = {
       val newTarget = if (location != null) {
         val target = new URI(location)
-        // The target path should already be encoded, so don't re-encode it, just the
-        // proxy address part.
-        val proxyBase = UIUtils.uiRoot(req)
-        val proxyPrefix = if (proxyBase.nonEmpty) s"$proxyUri$proxyBase" else proxyUri
-        s"${res.encodeURL(proxyPrefix)}${target.getPath()}"
+        val targetPath = target.getPath()
+        // UIUtils.uiRoot is the authoritative source for the configured base path:
+        // it checks spark.ui.proxyBase sys prop, APPLICATION_WEB_PROXY_BASE env var,
+        // X-Forwarded-Context header, and servlet context attribute -- in that order.
+        // req.getContextPath is only set after a context is matched, so it can be empty
+        // for Jetty's own pre-dispatch trailing-slash redirects. Use uiRoot first.
+        val uiRootBase = UIUtils.uiRoot(req)
+        val contextBasePath = if (uiRootBase.nonEmpty) {
+          uiRootBase
+        } else {
+          Option(req.getContextPath).filter(_.nonEmpty).getOrElse("")
+        }
+        val result = if (contextBasePath.nonEmpty && targetPath.startsWith(contextBasePath)) {
+          // Redirect target already contains the basePath -- just prepend the proxy host
+          s"${res.encodeURL(proxyUri)}$targetPath"
+        } else {
+          // Jetty generated a short redirect (e.g. /jobs/) missing the basePath --
+          // prepend proxyUri + basePath
+          val proxyPrefix = if (contextBasePath.nonEmpty) {
+            s"$proxyUri$contextBasePath"
+          } else {
+            proxyUri
+          }
+          s"${res.encodeURL(proxyPrefix)}$targetPath"
+        }
+        result
       } else {
         null
       }
+      logDebug(s"ProxyRedirect: '$location' -> '$newTarget'")
       super.sendRedirect(newTarget)
     }
   }

core/src/main/scala/org/apache/spark/ui/SparkUI.scala

@@ -94,6 +94,9 @@ private[spark] class SparkUI private (
 
   /** Initialize all components of the server. */
   def initialize(): Unit = {
+    if (basePath.nonEmpty) {
+      logInfo(s"Initializing SparkUI with basePath='$basePath'")
+    }
     val jobsTab = new JobsTab(this, store)
     attachTab(jobsTab)
     val stagesTab = new StagesTab(this, store)
@@ -103,17 +106,18 @@ private[spark] class SparkUI private (
     attachTab(new ExecutorsTab(this))
     addStaticHandler(SparkUI.STATIC_RESOURCE_DIR)
     attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath))
-    attachHandler(ApiRootResource.getServletHandler(this))
+    attachHandler(ApiRootResource.getServletHandler(this, basePath))
     if (sc.map(_.conf.get(UI_PROMETHEUS_ENABLED)).getOrElse(false)) {
       attachHandler(PrometheusResource.getServletHandler(this))
     }
 
     // These should be POST only, but, the YARN AM proxy won't proxy POSTs
     attachHandler(createRedirectHandler(
-      "/jobs/job/kill", "/jobs/", jobsTab.handleKillRequest, httpMethods = Set("GET", "POST")))
+      "/jobs/job/kill", "/jobs/", jobsTab.handleKillRequest,
+      basePath = basePath, httpMethods = Set("GET", "POST")))
     attachHandler(createRedirectHandler(
       "/stages/stage/kill", "/stages/", stagesTab.handleKillRequest,
-      httpMethods = Set("GET", "POST")))
+      basePath = basePath, httpMethods = Set("GET", "POST")))
   }
 
   initialize()

core/src/main/scala/org/apache/spark/ui/UIUtils.scala

@@ -209,18 +209,30 @@ private[spark] object UIUtils extends Logging {
   def uiRoot(request: HttpServletRequest): String = {
     // Knox uses X-Forwarded-Context to notify the application the base path
     val knoxBasePath = Option(request.getHeader("X-Forwarded-Context"))
+    // Check servlet context attribute set by JettyUtils when basePath is configured
+    val servletContextBasePath = Option(request.getServletContext)
+      .flatMap(ctx => Option(ctx.getAttribute(JettyUtils.PROXY_BASE_PATH_ATTRIBUTE)))
+      .map(_.toString)
     // SPARK-11484 - Use the proxyBase set by the AM, if not found then use env.
-    sys.props.get("spark.ui.proxyBase")
+    val root = sys.props.get("spark.ui.proxyBase")
       .orElse(sys.env.get("APPLICATION_WEB_PROXY_BASE"))
       .orElse(knoxBasePath)
+      .orElse(servletContextBasePath)
       .getOrElse("")
+    root
   }
 
   def prependBaseUri(
       request: HttpServletRequest,
       basePath: String = "",
       resource: String = ""): String = {
-    uiRoot(request) + basePath + resource
+    val root = uiRoot(request)
+    val result = if (root.nonEmpty && basePath.startsWith(root)) {
+      basePath + resource
+    } else {
+      root + basePath + resource
+    }
+    result
   }
 
   def commonHeaderNodes(request: HttpServletRequest): Seq[Node] = {
@@ -287,10 +299,17 @@ private[spark] object UIUtils extends Logging {
     }
     val helpButton: Seq[Node] = helpText.map(tooltip(_, "top")).getOrElse(Seq.empty)
 
+    val root = uiRoot(request)
+    val jsAppBasePath = if (root.nonEmpty && activeTab.basePath.startsWith(root)) {
+      ""
+    } else {
+      activeTab.basePath
+    }
+
     <html>
       <head>
         {commonHeaderNodes(request)}
-        <script>setAppBasePath('{activeTab.basePath}')</script>
+        <script>setAppBasePath('{jsAppBasePath}')</script>
         {if (showVisualization) vizHeaderNodes(request) else Seq.empty}
         {if (useDataTables) dataTablesHeaderNodes(request) else Seq.empty}
         <link rel="shortcut icon"

core/src/main/scala/org/apache/spark/ui/WebUI.scala

@@ -133,7 +133,8 @@ private[spark] abstract class WebUI(
    * @param path Path in UI where to mount the resources.
    */
   def addStaticHandler(resourceBase: String, path: String = "/static"): Unit = {
-    attachHandler(JettyUtils.createStaticHandler(resourceBase, path))
+    val handler = JettyUtils.createStaticHandler(resourceBase, path, basePath)
+    attachHandler(handler)
   }
 
   /** A hook to initialize components of the UI */