feat: support sqlite credential helper (#857)

Signed-off-by: Grant Linville <grant@acorn.io>

Author: g-linville

pkg/config/cliconfig.go

@@ -15,13 +15,32 @@ import (
 	"github.com/docker/cli/cli/config/types"
 )
 
+const (
+	WincredCredHelper       = "wincred"
+	OsxkeychainCredHelper   = "osxkeychain"
+	SecretserviceCredHelper = "secretservice"
+	PassCredHelper          = "pass"
+	FileCredHelper          = "file"
+	SqliteCredHelper        = "sqlite"
+
+	GPTScriptHelperPrefix = "gptscript-credential-"
+)
+
 var (
-	darwinHelpers  = []string{"osxkeychain", "file"}
-	windowsHelpers = []string{"wincred", "file"}
-	linuxHelpers   = []string{"secretservice", "pass", "file"}
+	darwinHelpers  = []string{OsxkeychainCredHelper, FileCredHelper, SqliteCredHelper}
+	windowsHelpers = []string{WincredCredHelper, FileCredHelper}
+	linuxHelpers   = []string{SecretserviceCredHelper, PassCredHelper, FileCredHelper, SqliteCredHelper}
 )
 
-const GPTScriptHelperPrefix = "gptscript-credential-"
+func listAsString(helpers []string) string {
+	if len(helpers) == 0 {
+		return ""
+	} else if len(helpers) == 1 {
+		return helpers[0]
+	}
+
+	return strings.Join(helpers[:len(helpers)-1], ", ") + " or " + helpers[len(helpers)-1]
+}
 
 type AuthConfig types.AuthConfig
 
@@ -150,13 +169,13 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) {
 		errMsg := fmt.Sprintf("invalid credential store '%s'", result.CredentialsStore)
 		switch runtime.GOOS {
 		case "darwin":
-			errMsg += " (use 'osxkeychain' or 'file')"
+			errMsg += fmt.Sprintf(" (use %s)", listAsString(darwinHelpers))
 		case "windows":
-			errMsg += " (use 'wincred' or 'file')"
+			errMsg += fmt.Sprintf(" (use %s)", listAsString(windowsHelpers))
 		case "linux":
-			errMsg += " (use 'secretservice', 'pass', or 'file')"
+			errMsg += fmt.Sprintf(" (use %s)", listAsString(linuxHelpers))
 		default:
-			errMsg += " (use 'file')"
+			errMsg += " (use file)"
 		}
 		errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.location)
 
@@ -169,11 +188,11 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) {
 func (c *CLIConfig) setDefaultCredentialsStore() error {
 	switch runtime.GOOS {
 	case "darwin":
-		c.CredentialsStore = "osxkeychain"
+		c.CredentialsStore = OsxkeychainCredHelper
 	case "windows":
-		c.CredentialsStore = "wincred"
+		c.CredentialsStore = WincredCredHelper
 	default:
-		c.CredentialsStore = "file"
+		c.CredentialsStore = FileCredHelper
 	}
 	return c.Save()
 }
@@ -187,7 +206,7 @@ func isValidCredentialHelper(helper string) bool {
 	case "linux":
 		return slices.Contains(linuxHelpers, helper)
 	default:
-		return helper == "file"
+		return helper == FileCredHelper
 	}
 }
 

pkg/credentials/store.go

@@ -46,7 +46,7 @@ func NewStore(cfg *config.CLIConfig, credentialBuilder CredentialBuilder, credCt
 	return Store{
 		credCtxs:       credCtxs,
 		credBuilder:    credentialBuilder,
-		credHelperDirs: GetCredentialHelperDirs(cacheDir),
+		credHelperDirs: GetCredentialHelperDirs(cacheDir, cfg.CredentialsStore),
 		cfg:            cfg,
 	}, nil
 }
@@ -199,7 +199,7 @@ func (s *Store) getStore(ctx context.Context) (credentials.Store, error) {
 }
 
 func (s *Store) getStoreByHelper(ctx context.Context, helper string) (credentials.Store, error) {
-	if helper == "" || helper == config.GPTScriptHelperPrefix+"file" {
+	if helper == "" || helper == config.GPTScriptHelperPrefix+config.FileCredHelper {
 		return credentials.NewFileStore(s.cfg), nil
 	}
 

pkg/credentials/util.go

@@ -1,18 +1,43 @@
 package credentials
 
 import (
+	"fmt"
 	"path/filepath"
+
+	"github.com/gptscript-ai/gptscript/pkg/config"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 )
 
 type CredentialHelperDirs struct {
 	RevisionFile, LastCheckedFile, BinDir string
 }
 
-func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs {
+func RepoNameForCredentialStore(store string) string {
+	switch store {
+	case config.SqliteCredHelper:
+		return "gptscript-credential-sqlite"
+	default:
+		return "gptscript-credential-helpers"
+	}
+}
+
+func GitURLForRepoName(repoName string) (string, error) {
+	switch repoName {
+	case "gptscript-credential-sqlite":
+		return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_SQLITE_ROOT", "https://github.com/gptscript-ai/gptscript-credential-sqlite.git"), nil
+	case "gptscript-credential-helpers":
+		return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), nil
+	default:
+		return "", fmt.Errorf("unknown repo name: %s", repoName)
+	}
+}
+
+func GetCredentialHelperDirs(cacheDir, store string) CredentialHelperDirs {
+	repoName := RepoNameForCredentialStore(store)
 	return CredentialHelperDirs{
-		RevisionFile:    filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"),
-		LastCheckedFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "last-checked"),
-		BinDir:          filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"),
+		RevisionFile:    filepath.Join(cacheDir, "repos", repoName, "revision"),
+		LastCheckedFile: filepath.Join(cacheDir, "repos", repoName, "last-checked"),
+		BinDir:          filepath.Join(cacheDir, "repos", repoName, "bin"),
 	}
 }
 

pkg/repos/get.go

@@ -16,7 +16,6 @@ import (
 	"github.com/BurntSushi/locker"
 	"github.com/gptscript-ai/gptscript/pkg/config"
 	"github.com/gptscript-ai/gptscript/pkg/credentials"
-	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/git"
 	"github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang"
@@ -55,10 +54,10 @@ func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []str
 }
 
 type Manager struct {
+	cacheDir         string
 	storageDir       string
 	gitDir           string
 	runtimeDir       string
-	credHelperDirs   credentials.CredentialHelperDirs
 	runtimes         []Runtime
 	credHelperConfig *credHelperConfig
 }
@@ -72,11 +71,11 @@ type credHelperConfig struct {
 func New(cacheDir string, runtimes ...Runtime) *Manager {
 	root := filepath.Join(cacheDir, "repos")
 	return &Manager{
-		storageDir:     root,
-		gitDir:         filepath.Join(root, "git"),
-		runtimeDir:     filepath.Join(root, "runtimes"),
-		credHelperDirs: credentials.GetCredentialHelperDirs(cacheDir),
-		runtimes:       runtimes,
+		cacheDir:   cacheDir,
+		storageDir: root,
+		gitDir:     filepath.Join(root, "git"),
+		runtimeDir: filepath.Join(root, "runtimes"),
+		runtimes:   runtimes,
 	}
 }
 
@@ -110,50 +109,59 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 		distInfo, suffix string
 	)
 	// The file helper is built-in and does not need to be downloaded.
-	if helperName == "file" {
+	if helperName == config.FileCredHelper {
 		return nil
 	}
 	switch helperName {
-	case "wincred":
+	case config.WincredCredHelper:
 		suffix = ".exe"
 	default:
 		distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH)
 	}
 
-	locker.Lock("gptscript-credential-helpers")
-	defer locker.Unlock("gptscript-credential-helpers")
+	repoName := credentials.RepoNameForCredentialStore(helperName)
+
+	locker.Lock(repoName)
+	defer locker.Unlock(repoName)
+
+	credHelperDirs := credentials.GetCredentialHelperDirs(m.cacheDir, helperName)
 
 	// Load the last-checked file to make sure we haven't checked the repo in the last 24 hours.
 	now := time.Now()
-	lastChecked, err := os.ReadFile(m.credHelperDirs.LastCheckedFile)
+	lastChecked, err := os.ReadFile(credHelperDirs.LastCheckedFile)
 	if err == nil {
 		if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(lastChecked))); err == nil && now.Sub(t) < 24*time.Hour {
 			// Make sure the binary still exists, and if it does, return.
-			if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
+			if _, err := os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
 				log.Debugf("Credential helper %s up-to-date as of %v, checking for updates after %v", helperName, t, t.Add(24*time.Hour))
 				return nil
 			}
 		}
 	}
 
-	if err := os.MkdirAll(filepath.Dir(m.credHelperDirs.LastCheckedFile), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(credHelperDirs.LastCheckedFile), 0755); err != nil {
 		return err
 	}
 
 	// Update the last-checked file.
-	if err := os.WriteFile(m.credHelperDirs.LastCheckedFile, []byte(now.Format(time.RFC3339)), 0644); err != nil {
+	if err := os.WriteFile(credHelperDirs.LastCheckedFile, []byte(now.Format(time.RFC3339)), 0644); err != nil {
+		return err
+	}
+
+	gitURL, err := credentials.GitURLForRepoName(repoName)
+	if err != nil {
 		return err
 	}
 
 	tool := types.Tool{
 		ToolDef: types.ToolDef{
 			Parameters: types.Parameters{
-				Name: "gptscript-credential-helpers",
+				Name: repoName,
 			},
 		},
 		Source: types.ToolSource{
 			Repo: &types.Repo{
-				Root: runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"),
+				Root: gitURL,
 			},
 		},
 	}
@@ -164,12 +172,12 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 
 	var needsDownloaded bool
 	// Check the last revision shasum and see if it is different from the current one.
-	lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile)
+	lastRevision, err := os.ReadFile(credHelperDirs.RevisionFile)
 	if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) {
 		// Need to pull the latest version.
 		needsDownloaded = true
 		// Update the revision file to the new revision.
-		if err = os.WriteFile(m.credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil {
+		if err = os.WriteFile(credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil {
 			return err
 		}
 	} else if err != nil {
@@ -179,15 +187,15 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co
 	if !needsDownloaded {
 		// Check for the existence of the credential helper binary.
 		// If it's there, we have no need to download it and can just return.
-		if _, err = os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
+		if _, err = os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil {
 			return nil
 		}
 	}
 
 	// Find the Go runtime and use it to build the credential helper.
 	for _, rt := range m.runtimes {
 		if strings.HasPrefix(rt.ID(), "go") {
-			return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, m.credHelperDirs.BinDir)
+			return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, credHelperDirs.BinDir)
 		}
 	}
 

pkg/repos/runtimes/golang/golang.go

@@ -18,6 +18,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/gptscript-ai/gptscript/pkg/config"
 	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
 	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
@@ -286,7 +287,7 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource
 }
 
 func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error {
-	if helperName == "file" {
+	if helperName == config.FileCredHelper {
 		return nil
 	}