feat: add system-tools-dir for system managed tools

ibuildthecloud · ibuildthecloud · commit 2d1632ceaecb · 2024-10-25T16:06:10.000-07:00
diff --git a/docs/docs/04-command-line-reference/gptscript.md b/docs/docs/04-command-line-reference/gptscript.md
@@ -43,6 +43,7 @@ gptscript [flags] PROGRAM_FILE [INPUT...]
   -q, --quiet                               No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
       --save-chat-state-file string         A file to save the chat state to so that a conversation can be resumed with --chat-state ($GPTSCRIPT_SAVE_CHAT_STATE_FILE)
       --sub-tool string                     Use tool of this name, not the first tool in file ($GPTSCRIPT_SUB_TOOL)
+      --system-tools-dir string             Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR)
       --ui                                  Launch the UI ($GPTSCRIPT_UI)
       --workspace string                    Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
 ```
diff --git a/docs/docs/04-command-line-reference/gptscript_eval.md b/docs/docs/04-command-line-reference/gptscript_eval.md
@@ -46,6 +46,7 @@ gptscript eval [flags]
       --openai-org-id string            OpenAI organization ID ($OPENAI_ORG_ID)
   -o, --output string                   Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
   -q, --quiet                           No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
+      --system-tools-dir string         Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR)
       --workspace string                Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
 ```
 
diff --git a/docs/docs/04-command-line-reference/gptscript_fmt.md b/docs/docs/04-command-line-reference/gptscript_fmt.md
@@ -40,6 +40,7 @@ gptscript fmt [flags]
       --openai-org-id string            OpenAI organization ID ($OPENAI_ORG_ID)
   -o, --output string                   Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
   -q, --quiet                           No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
+      --system-tools-dir string         Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR)
       --workspace string                Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
 ```
 
diff --git a/docs/docs/04-command-line-reference/gptscript_getenv.md b/docs/docs/04-command-line-reference/gptscript_getenv.md
@@ -39,6 +39,7 @@ gptscript getenv [flags] KEY [DEFAULT]
       --openai-org-id string            OpenAI organization ID ($OPENAI_ORG_ID)
   -o, --output string                   Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
   -q, --quiet                           No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
+      --system-tools-dir string         Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR)
       --workspace string                Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
 ```
 
diff --git a/docs/docs/04-command-line-reference/gptscript_parse.md b/docs/docs/04-command-line-reference/gptscript_parse.md
@@ -40,6 +40,7 @@ gptscript parse [flags]
       --openai-org-id string            OpenAI organization ID ($OPENAI_ORG_ID)
   -o, --output string                   Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT)
   -q, --quiet                           No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET)
+      --system-tools-dir string         Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR)
       --workspace string                Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE)
 ```
 
diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go
@@ -48,7 +48,7 @@ func (c *Credential) Run(cmd *cobra.Command, _ []string) error {
 	}
 	opts = gptscript.Complete(opts)
 	if opts.Runner.RuntimeManager == nil {
-		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
+		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)
 	}
 
 	ctxs := opts.CredentialContexts
diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go
@@ -35,7 +35,7 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error {
 
 	opts = gptscript.Complete(opts)
 	if opts.Runner.RuntimeManager == nil {
-		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
+		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)
 	}
 
 	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {
diff --git a/pkg/cli/credential_show.go b/pkg/cli/credential_show.go
@@ -37,7 +37,7 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error {
 
 	opts = gptscript.Complete(opts)
 	if opts.Runner.RuntimeManager == nil {
-		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)
+		opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)
 	}
 
 	if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {
diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go
@@ -47,6 +47,7 @@ type GPTScript struct {
 	CacheOptions
 	OpenAIOptions
 	DisplayOptions
+	SystemToolsDir string `usage:"Directory that contains system managed tool for which GPTScript will not manage the runtime"`
 	Color          *bool  `usage:"Use color in output (default true)" default:"true"`
 	Confirm        bool   `usage:"Prompt before running potentially dangerous commands"`
 	Debug          bool   `usage:"Enable debug logging"`
@@ -146,6 +147,7 @@ func (r *GPTScript) NewGPTScriptOpts() (gptscript.Options, error) {
 		Workspace:            r.Workspace,
 		DisablePromptServer:  r.UI,
 		DefaultModelProvider: r.DefaultModelProvider,
+		SystemToolsDir:       r.SystemToolsDir,
 	}
 
 	if r.Confirm {
diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go
@@ -52,6 +52,7 @@ type Options struct {
 	Quiet                *bool
 	Workspace            string
 	DisablePromptServer  bool
+	SystemToolsDir       string
 	Env                  []string
 }
 
@@ -63,6 +64,7 @@ func Complete(opts ...Options) Options {
 		result.Runner = runner.Complete(result.Runner, opt.Runner)
 		result.OpenAI = openai.Complete(result.OpenAI, opt.OpenAI)
 
+		result.SystemToolsDir = types.FirstSet(opt.SystemToolsDir, result.SystemToolsDir)
 		result.CredentialContexts = opt.CredentialContexts
 		result.Quiet = types.FirstSet(opt.Quiet, result.Quiet)
 		result.Workspace = types.FirstSet(opt.Workspace, result.Workspace)
@@ -99,7 +101,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) {
 	}
 
 	if opts.Runner.RuntimeManager == nil {
-		opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir())
+		opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir(), opts.SystemToolsDir)
 	}
 
 	if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil {
diff --git a/pkg/repos/get.go b/pkg/repos/get.go
@@ -8,6 +8,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -58,7 +59,7 @@ type Manager struct {
 	storageDir       string
 	gitDir           string
 	runtimeDir       string
-	systemDir        string
+	systemDirs       []string
 	runtimes         []Runtime
 	credHelperConfig *credHelperConfig
 }
@@ -69,14 +70,22 @@ type credHelperConfig struct {
 	cliCfg      *config.CLIConfig
 }
 
-func New(cacheDir string, runtimes ...Runtime) *Manager {
-	root := filepath.Join(cacheDir, "repos")
+func New(cacheDir, systemDir string, runtimes ...Runtime) *Manager {
+	var (
+		systemDirs []string
+		root       = filepath.Join(cacheDir, "repos")
+	)
+
+	if strings.TrimSpace(systemDir) != "" {
+		systemDirs = regexp.MustCompile("[;:,]").Split(strings.TrimSpace(systemDir), -1)
+	}
+
 	return &Manager{
 		cacheDir:   cacheDir,
 		storageDir: root,
 		gitDir:     filepath.Join(root, "git"),
 		runtimeDir: filepath.Join(root, "runtimes"),
-		systemDir:  filepath.Join(root, "system"),
+		systemDirs: systemDirs,
 		runtimes:   runtimes,
 	}
 }
@@ -273,8 +282,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 }
 
 func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) {
-	if strings.HasPrefix(tool.WorkingDir, m.systemDir) {
-		return tool.WorkingDir, env, nil
+	for _, systemDir := range m.systemDirs {
+		if strings.HasPrefix(tool.WorkingDir, systemDir) {
+			return tool.WorkingDir, env, nil
+		}
 	}
 
 	var isLocal bool
diff --git a/pkg/repos/get_test.go b/pkg/repos/get_test.go
@@ -19,7 +19,7 @@ var (
 )
 
 func TestManager_GetContext(t *testing.T) {
-	m := New(testCacheHome, &python.Runtime{
+	m := New(testCacheHome, "", &python.Runtime{
 		Version: "3.11",
 	})
 	cwd, env, err := m.GetContext(context.Background(), types.Tool{
diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go
@@ -30,6 +30,6 @@ var Runtimes = []repos.Runtime{
 	},
 }
 
-func Default(cacheDir string) engine.RuntimeManager {
-	return repos.New(cacheDir, Runtimes...)
+func Default(cacheDir, systemDir string) engine.RuntimeManager {
+	return repos.New(cacheDir, systemDir, Runtimes...)
 }
diff --git a/pkg/sdkserver/server.go b/pkg/sdkserver/server.go
@@ -111,7 +111,7 @@ func run(ctx context.Context, listener net.Listener, opts Options) error {
 		workspaceTool:    opts.WorkspaceTool,
 		client:           g,
 		events:           events,
-		runtimeManager:   runtimes.Default(opts.Options.Cache.CacheDir), // TODO - do we always want to use runtimes.Default here?
+		runtimeManager:   runtimes.Default(opts.Options.Cache.CacheDir, opts.SystemToolsDir),
 		waitingToConfirm: make(map[string]chan runner.AuthorizerResponse),
 		waitingToPrompt:  make(map[string]chan map[string]string),
 	}
diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go
@@ -196,7 +196,7 @@ func NewRunner(t *testing.T) *Runner {
 	cacheDir, err := xdg.CacheFile("gptscript-test-cache/runtime")
 	require.NoError(t, err)
 
-	rm := runtimes.Default(cacheDir)
+	rm := runtimes.Default(cacheDir, "")
 
 	run, err := runner.New(c, credentials.NoopStore{}, runner.Options{
 		Sequential:     true,

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ func (c Credential) Run(cmd cobra.Command, _ []string) error {`
`48`	`48`	`}`
`49`	`49`	`opts = gptscript.Complete(opts)`
`50`	`50`	`if opts.Runner.RuntimeManager == nil {`
`51`		`- opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)`
	`51`	`+ opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`ctxs := opts.CredentialContexts`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ func (c Delete) Run(cmd cobra.Command, args []string) error {`
`35`	`35`
`36`	`36`	`opts = gptscript.Complete(opts)`
`37`	`37`	`if opts.Runner.RuntimeManager == nil {`
`38`		`- opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)`
	`38`	`+ opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)`
`39`	`39`	`}`
`40`	`40`
`41`	`41`	`if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ func (c Show) Run(cmd cobra.Command, args []string) error {`
`37`	`37`
`38`	`38`	`opts = gptscript.Complete(opts)`
`39`	`39`	`if opts.Runner.RuntimeManager == nil {`
`40`		`- opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir)`
	`40`	`+ opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir)`
`41`	`41`	`}`
`42`	`42`
`43`	`43`	`if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,7 @@ type Options struct {`
`52`	`52`	`Quiet *bool`
`53`	`53`	`Workspace string`
`54`	`54`	`DisablePromptServer bool`
	`55`	`+ SystemToolsDir string`
`55`	`56`	`Env []string`
`56`	`57`	`}`
`57`	`58`
`@@ -63,6 +64,7 @@ func Complete(opts ...Options) Options {`
`63`	`64`	`result.Runner = runner.Complete(result.Runner, opt.Runner)`
`64`	`65`	`result.OpenAI = openai.Complete(result.OpenAI, opt.OpenAI)`
`65`	`66`
	`67`	`+ result.SystemToolsDir = types.FirstSet(opt.SystemToolsDir, result.SystemToolsDir)`
`66`	`68`	`result.CredentialContexts = opt.CredentialContexts`
`67`	`69`	`result.Quiet = types.FirstSet(opt.Quiet, result.Quiet)`
`68`	`70`	`result.Workspace = types.FirstSet(opt.Workspace, result.Workspace)`
`@@ -99,7 +101,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) {`
`99`	`101`	`}`
`100`	`102`
`101`	`103`	`if opts.Runner.RuntimeManager == nil {`
`102`		`- opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir())`
	`104`	`+ opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir(), opts.SystemToolsDir)`
`103`	`105`	`}`
`104`	`106`
`105`	`107`	`if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil {`