Support non-Homebrew OpenSSL locations on macOS

[TyceHerrman]

Written with support from Claude Code, but reviewed by me.

Problem

The plugin hardcodes Homebrew's OpenSSL path ($HOMEBREW_PREFIX/opt/openssl), which fails for users using other install methods for openssl like MacPorts, mise-nix backend, or manual installations.

Why this matters

Any macOS user who doesn't use Homebrew for OpenSSL—whether they prefer mise-nix for hermetic builds, MacPorts for their package management, or have OpenSSL installed manually or via other tools—hits a build failure when installing PostgreSQL. The only workaround is installing Homebrew's OpenSSL alongside their preferred installation.

Current: only checks Homebrew

local openssl_path = homebrew_prefix .. "/opt/openssl"
f = io.open(openssl_path .. "/lib", "r")  -- weak validation

Proposed Solution

Add fallback detection with proper validation:

local function file_exists(path)
    local f = io.open(path, "r")
    if f then f:close() return true end
    return false
end

local function validate_openssl(prefix)
    if not prefix then return false end
    -- Require BOTH header (for compilation) and library
    return file_exists(prefix .. "/include/openssl/ssl.h") and
           (file_exists(prefix .. "/lib/libssl.dylib") or
            file_exists(prefix .. "/lib/libssl.a"))
end

local function pkg_config_openssl()
    local handle = io.popen("pkg-config --variable=prefix openssl 2>/dev/null")
    if not handle then return nil end
    local prefix = handle:read("*l")
    handle:close()
    if prefix and prefix ~= "" and prefix:match("^/") then
        return prefix
    end
    return nil
end

local function find_nix_openssl()
    local nix_ssl_cert = os.getenv("NIX_SSL_CERT_FILE")
    if nix_ssl_cert then
        local nix_prefix = nix_ssl_cert:match("(/nix/store/[^/]+-openssl[^/]*)")
        if nix_prefix and validate_openssl(nix_prefix) then
            return nix_prefix
        end
    end
    local nix_paths = {
        os.getenv("HOME") .. "/.nix-profile",
        "/nix/var/nix/profiles/default",
    }
    for _, path in ipairs(nix_paths) do
        if validate_openssl(path) then return path end
    end
    return nil
end

local function find_openssl()
    -- 1. Environment override
    for _, var in ipairs({"OPENSSL_ROOT_DIR", "OPENSSL_DIR"}) do
        local path = os.getenv(var)
        if validate_openssl(path) then return path end
    end

      -- 2. pkg-config (works when environment is activated)
      local pkg_path = pkg_config_openssl()
      if validate_openssl(pkg_path) then return pkg_path end

      -- 3. Nix paths (for impure environments outside nix-shell)
      local nix_path = find_nix_openssl()
      if nix_path then return nix_path end

      -- 4. Homebrew (versioned first)
      local brew = os.getenv("HOMEBREW_PREFIX") or "/opt/homebrew"
      for _, path in ipairs({brew .. "/opt/openssl@3", brew .. "/opt/openssl"}) do
          if validate_openssl(path) then return path end
      end

      -- 5. Fallbacks
      for _, path in ipairs({"/usr/local/opt/openssl@3", "/usr/local/opt/openssl", "/opt/local", "/usr/local/ssl"}) do
          if validate_openssl(path) then return path end
      end

      return nil
  end

Key Changes

Aspect	Current	Proposed
Discovery	Homebrew only	env → pkg-config → Nix → Homebrew → fallbacks
Validation	Checks if /lib dir exists	Checks for ssl.h header + libssl library
User override	POSTGRES_CONFIGURE_OPTIONS only (replaces all auto-detection)	Adds OPENSSL_ROOT_DIR for
OpenSSL-only override

Notes

• Nix detection: Handles users with Nix-installed OpenSSL running outside nix-shell, where PKG_CONFIG_PATH isn't set.

• Add new check for Dev files (not strictly needed for this issue): The ssl.h check ensures OpenSSL development files are present (required for building PostgreSQL), not just runtime libs. If implemented, consider warning users like: "OpenSSL development files required (libssl-dev on Debian, openssl-devel on RHEL)."

Environment

• mise 2026.2.1, macOS Sequoia (arm64)