Feature/dns proxy healthcheck tnz 66259

jobs/proxy/spec

@@ -8,6 +8,7 @@ templates:
   drain.erb: bin/drain
   healthcheck.erb: bin/healthcheck
   prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+  dns_healthcheck.erb: bin/dns/healthy
 
 consumes:
 - name: proxy

jobs/proxy/templates/dns_healthcheck.erb

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+export TIMEOUT=3s
+export PORT="<%= p('port') %>"
+
+/var/vcap/packages/proxy/bin/pingdb

packages/proxy/packaging

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 
-set -eu
+set -o errexit -o nounset -o pipefail
 
 source /var/vcap/packages/golang-1-linux/bosh/compile.env
 
-cd github.com/cloudfoundry-incubator/switchboard
-  go build -mod=vendor -o "${BOSH_INSTALL_TARGET}/bin/proxy"
-  cp -r static "${BOSH_INSTALL_TARGET}/static"
-cd -
+export GOBIN=${BOSH_INSTALL_TARGET}/bin
+go -C github.com/cloudfoundry-incubator/switchboard install -mod=vendor ./cmd/...
+cp -r github.com/cloudfoundry-incubator/switchboard/static "${BOSH_INSTALL_TARGET}/static"

packages/proxy/spec

@@ -3,4 +3,4 @@ name: proxy
 dependencies:
   - golang-1-linux
 files:
-  - github.com/cloudfoundry-incubator/switchboard/**
+  - github.com/cloudfoundry-incubator/switchboard/**

src/e2e-tests/proxy_healthcheck_test.go

@@ -0,0 +1,177 @@
+package e2e_tests
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"e2e-tests/utilities/bosh"
+)
+
+var _ = Describe("Proxy healthcheck", Ordered, Label("proxy", "healthchecks"), func() {
+	var (
+		deploymentName string
+		proxyIPs       []string
+		proxies        []bosh.Instance
+		numProxies     int
+		proxyDNSQuery  string
+	)
+
+	BeforeAll(func() {
+		deploymentName = "pxc-proxy-healthcheck-" + uuid.New().String()
+
+		Expect(bosh.DeployPXC(deploymentName,
+			bosh.Operation(`use-clustered.yml`),
+			bosh.Operation(`iaas/cluster.yml`),
+			bosh.Operation(`require-tls.yml`),
+			bosh.Operation(`mysql-version.yml`),
+			bosh.Var("mysql_version", "8.4"),
+		)).To(Succeed())
+
+		DeferCleanup(func() {
+			if CurrentSpecReport().Failed() {
+				return
+			}
+			Expect(bosh.DeleteDeployment(deploymentName)).To(Succeed())
+		})
+
+		var err error
+		proxies, err = bosh.Instances(deploymentName, bosh.MatchByInstanceGroup("proxy"))
+		Expect(err).NotTo(HaveOccurred())
+		proxyIPs, err = bosh.InstanceIPs(deploymentName, bosh.MatchByInstanceGroup("proxy"))
+		Expect(err).NotTo(HaveOccurred())
+		numProxies = len(proxyIPs)
+
+		proxyDNSQuery = boshDNSAddress(deploymentName, "proxy")
+	})
+
+	It("deploys proxy healthcheck scripts", func() {
+		_, err := bosh.RemoteCommand(deploymentName, "proxy",
+			"ls -l /var/vcap/jobs/proxy/bin/dns/healthy")
+		Expect(err).NotTo(HaveOccurred(), "deployment missing expected healthcheck scripts")
+	})
+
+	It("reports all proxies healthy", func() {
+		var healthyProxyIPs []string
+		Eventually(
+			func() int {
+				healthyProxyIPs = interrogateDNS(deploymentName, "mysql/0", proxyDNSQuery)
+				return len(healthyProxyIPs)
+			}).WithTimeout(time.Second*30).WithPolling(time.Second*2).Should(Equal(numProxies), fmt.Sprintf("expected %d healthy proxies, got %d", numProxies, len(healthyProxyIPs)))
+	})
+
+	When("each proxy is temporarily unhealthy", func() {
+		DescribeTable("app connections bypass unhealthy proxies",
+			func(proxyID string) {
+
+				By("temporarily pausing proxy " + proxyID)
+				pausedProxyInstance := "proxy/" + proxyID
+				pauseProxy(deploymentName, pausedProxyInstance)
+				DeferCleanup(func() {
+					resumeProxy(deploymentName, pausedProxyInstance)
+				})
+
+				By("seeing DNS exclude that proxy from the healthy proxies")
+				var upProxyIPs, expectedProxyIPs []string
+				pausedIndex := slices.IndexFunc(proxies, func(s bosh.Instance) bool { return s.Index == proxyID })
+				Expect(pausedIndex).NotTo(Equal(-1), "unable to retrieve proxy/0 IP address")
+				pausedIP := proxies[pausedIndex].IP
+				for _, proxyIP := range proxyIPs {
+					if proxyIP == pausedIP {
+						continue
+					}
+					expectedProxyIPs = append(expectedProxyIPs, proxyIP)
+				}
+				Eventually(func() []string {
+					upProxyIPs = interrogateDNS(deploymentName, "mysql/0", proxyDNSQuery)
+					return upProxyIPs
+				}).WithTimeout(time.Second*30).WithPolling(time.Second*2).Should(ConsistOf(expectedProxyIPs),
+					fmt.Sprintf("expected DNS query to return expected proxy IPs %v, instead got %v", expectedProxyIPs, upProxyIPs))
+
+				By("ensuring bosh DNS bypasses that unhealthy proxy for incoming connections")
+				Expect(bosh.RunErrand(deploymentName, "smoke-tests",
+					"mysql/0")).To(Succeed(),
+					"smoke-tests unexpectedly failed while a proxy was still available")
+			},
+			Entry("when pausing proxy/0", "0"),
+			Entry("when pausing proxy/1", "1"),
+		)
+	})
+})
+
+// Halt the process "proxy" running on the provided instance
+func pauseProxy(deploymentName, instance string) {
+	GinkgoHelper()
+	_, err := bosh.RemoteCommand(deploymentName, instance,
+		"sudo pkill -SIGSTOP proxy")
+	Expect(err).NotTo(HaveOccurred())
+}
+func resumeProxy(deploymentName, instance string) {
+	GinkgoHelper()
+	_, err := bosh.RemoteCommand(deploymentName, instance,
+		"sudo pkill -SIGCONT proxy")
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func interrogateDNS(deploymentName, sourceInstance, dnsQuery string) []string {
+	GinkgoHelper()
+	output, err := bosh.RemoteCommand(deploymentName, sourceInstance, "dig +short "+dnsQuery)
+	Expect(err).NotTo(HaveOccurred(), "remote dig lookup %s failed: %w\n output: %s", dnsQuery, err, output)
+	return strings.Fields(output)
+}
+
+func boshDNSAddress(deploymentName, jobName string) string {
+	GinkgoHelper()
+
+	// Get all links
+	var deploymentLinks []struct {
+		ID   string `json:"id"`
+		Name string `json:"name"`
+	}
+
+	Expect(boshCurl("/links?deployment="+deploymentName, &deploymentLinks)).To(Succeed())
+
+	// Get first proxy link
+	var targetLinkID string
+	for _, link := range deploymentLinks {
+		// Get ID of our target job's first link
+		if link.Name == jobName {
+			targetLinkID = link.ID
+			break
+		}
+	}
+	Expect(targetLinkID).NotTo(BeEmpty(), "link ID not detected for job %s", jobName)
+
+	// Get links address
+	var address struct {
+		Address string `json:"address"`
+	}
+
+	Expect(boshCurl("/link_address?link_id="+targetLinkID, &address)).To(Succeed())
+
+	return address.Address
+}
+
+func boshCurl(endpoint string, target any) error {
+	var output bytes.Buffer
+	cmd := exec.Command("bosh", "curl", endpoint)
+	cmd.Stdout = &output
+	cmd.Stderr = GinkgoWriter
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("remote boshCurl failed: %w", err)
+	}
+
+	if err := json.Unmarshal(output.Bytes(), target); err != nil {
+		return fmt.Errorf("failed to parse JSON: %w", err)
+	}
+
+	return nil
+}

src/github.com/cloudfoundry-incubator/switchboard/cmd/pingdb/main.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+)
+
+func main() {
+	port := os.Getenv("PORT")
+	if port == "" {
+		fmt.Fprintf(os.Stderr, "Error: PORT environment variable is required\n")
+		os.Exit(1)
+	}
+
+	timeoutStr := os.Getenv("TIMEOUT")
+	if timeoutStr == "" {
+		fmt.Fprintf(os.Stderr, "Error: TIMEOUT environment variable is required\n")
+		os.Exit(1)
+	}
+
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil || timeout <= 0 {
+		fmt.Fprintf(os.Stderr, "Error: TIMEOUT must be a go parsable interval\n")
+		os.Exit(1)
+	}
+
+	// Hard-coded/invalid "pingdb" username/pw are sufficient to exercise db connectivity.
+	dsn := fmt.Sprintf("pingdb:pingdb@tcp(127.0.0.1:%s)/?tls=preferred", port)
+	db, err := sql.Open("mysql", dsn)
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "Error opening database connection: %v\n", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if closeErr := db.Close(); closeErr != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "Warning: Error closing database connection: %v\n", closeErr)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	// Ping using timeout context, and expect a recognized MySQL-generated error in response.
+	if err := db.PingContext(ctx); err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			// timed out
+			_, _ = fmt.Fprintf(os.Stderr, "Error: database ping timeout after %.0f seconds\n", timeout.Seconds())
+			os.Exit(1)
+		}
+
+		// Recognized MySQL errors do not trigger failure (since the proxy contacted the db).
+		if RecognizedMySQLError(err) {
+			os.Exit(0)
+		}
+
+		_, _ = fmt.Fprintf(os.Stderr, "Error: database ping: %v\n", err.Error())
+		os.Exit(2)
+	}
+
+	// Ping of user pingdb/pingdb succeeded.
+	os.Exit(0)
+}
+
+func RecognizedMySQLError(err error) bool {
+	var mysqlErr *mysql.MySQLError
+	if !errors.As(err, &mysqlErr) {
+		return false
+	}
+
+	// Test for MySQL Server errors (proof MySQL responded via the proxy)
+	// Server error ranges: 1000-1999, 3000-9999
+	// Client error range: 2000-2999 (connection issues, other local client errors)
+	//
+	// Common expected server errors:
+	// - 1045: Access denied (our invalid pingdb/pingdb credentials)
+	// - 1129: Host blocked
+	// - 1130: Host not allowed to connect
+	//
+	// Client errors we DON'T want to accept:
+	// - 2002: Can't connect (network issue)
+	// - 2003: Can't connect to server (connection refused)
+	// - 2005: Unknown host
+
+	return mysqlErr.Number < 2000 || mysqlErr.Number >= 3000
+}

src/github.com/cloudfoundry-incubator/switchboard/cmd/pingdb/main_suite_test.go

@@ -0,0 +1,29 @@
+package main_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+var (
+	pingdbPath string
+)
+
+func TestPingDB(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PingDB Suite")
+}
+
+var _ = BeforeSuite(func() {
+	var err error
+	pingdbPath, err = gexec.Build("github.com/cloudfoundry-incubator/switchboard/cmd/pingdb")
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	gexec.Kill()
+	gexec.CleanupBuildArtifacts()
+})