From fdb3ef18daf9ad0b4ce7f411a689314b920ef52b Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 25 Oct 2024 15:48:26 -0700 Subject: [PATCH] feat: add system-tools-dir for system managed tools --- .../04-command-line-reference/gptscript.md | 1 + .../gptscript_eval.md | 1 + .../gptscript_fmt.md | 1 + .../gptscript_getenv.md | 1 + .../gptscript_parse.md | 1 + pkg/cli/credential.go | 2 +- pkg/cli/credential_delete.go | 2 +- pkg/cli/credential_show.go | 2 +- pkg/cli/gptscript.go | 2 ++ pkg/gptscript/gptscript.go | 4 +++- pkg/repos/get.go | 23 ++++++++++++++----- pkg/repos/get_test.go | 2 +- pkg/repos/runtimes/default.go | 4 ++-- pkg/sdkserver/server.go | 2 +- pkg/tests/tester/runner.go | 2 +- 15 files changed, 35 insertions(+), 15 deletions(-) diff --git a/docs/docs/04-command-line-reference/gptscript.md b/docs/docs/04-command-line-reference/gptscript.md index 8a726c64..4ca35228 100644 --- a/docs/docs/04-command-line-reference/gptscript.md +++ b/docs/docs/04-command-line-reference/gptscript.md @@ -43,6 +43,7 @@ gptscript [flags] PROGRAM_FILE [INPUT...] -q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET) --save-chat-state-file string A file to save the chat state to so that a conversation can be resumed with --chat-state ($GPTSCRIPT_SAVE_CHAT_STATE_FILE) --sub-tool string Use tool of this name, not the first tool in file ($GPTSCRIPT_SUB_TOOL) + --system-tools-dir string Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR) --ui Launch the UI ($GPTSCRIPT_UI) --workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE) ``` diff --git a/docs/docs/04-command-line-reference/gptscript_eval.md b/docs/docs/04-command-line-reference/gptscript_eval.md index 257cf609..ddbecc9f 100644 --- a/docs/docs/04-command-line-reference/gptscript_eval.md +++ b/docs/docs/04-command-line-reference/gptscript_eval.md @@ -46,6 +46,7 @@ gptscript eval [flags] --openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID) -o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT) -q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET) + --system-tools-dir string Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR) --workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE) ``` diff --git a/docs/docs/04-command-line-reference/gptscript_fmt.md b/docs/docs/04-command-line-reference/gptscript_fmt.md index 1175a1f1..2b042623 100644 --- a/docs/docs/04-command-line-reference/gptscript_fmt.md +++ b/docs/docs/04-command-line-reference/gptscript_fmt.md @@ -40,6 +40,7 @@ gptscript fmt [flags] --openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID) -o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT) -q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET) + --system-tools-dir string Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR) --workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE) ``` diff --git a/docs/docs/04-command-line-reference/gptscript_getenv.md b/docs/docs/04-command-line-reference/gptscript_getenv.md index 4a688439..7e677c5c 100644 --- a/docs/docs/04-command-line-reference/gptscript_getenv.md +++ b/docs/docs/04-command-line-reference/gptscript_getenv.md @@ -39,6 +39,7 @@ gptscript getenv [flags] KEY [DEFAULT] --openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID) -o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT) -q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET) + --system-tools-dir string Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR) --workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE) ``` diff --git a/docs/docs/04-command-line-reference/gptscript_parse.md b/docs/docs/04-command-line-reference/gptscript_parse.md index 66d2791c..567b0c05 100644 --- a/docs/docs/04-command-line-reference/gptscript_parse.md +++ b/docs/docs/04-command-line-reference/gptscript_parse.md @@ -40,6 +40,7 @@ gptscript parse [flags] --openai-org-id string OpenAI organization ID ($OPENAI_ORG_ID) -o, --output string Save output to a file, or - for stdout ($GPTSCRIPT_OUTPUT) -q, --quiet No output logging (set --quiet=false to force on even when there is no TTY) ($GPTSCRIPT_QUIET) + --system-tools-dir string Directory that contains system managed tool for which GPTScript will not manage the runtime ($GPTSCRIPT_SYSTEM_TOOLS_DIR) --workspace string Directory to use for the workspace, if specified it will not be deleted on exit ($GPTSCRIPT_WORKSPACE) ``` diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go index 674160b9..a46c483b 100644 --- a/pkg/cli/credential.go +++ b/pkg/cli/credential.go @@ -48,7 +48,7 @@ func (c *Credential) Run(cmd *cobra.Command, _ []string) error { } opts = gptscript.Complete(opts) if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) + opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) } ctxs := opts.CredentialContexts diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go index b17ae851..81392f36 100644 --- a/pkg/cli/credential_delete.go +++ b/pkg/cli/credential_delete.go @@ -35,7 +35,7 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error { opts = gptscript.Complete(opts) if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) + opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) } if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { diff --git a/pkg/cli/credential_show.go b/pkg/cli/credential_show.go index d8ea980b..ab2e9cd1 100644 --- a/pkg/cli/credential_show.go +++ b/pkg/cli/credential_show.go @@ -37,7 +37,7 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error { opts = gptscript.Complete(opts) if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) + opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) } if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index 66719adc..d0481ec8 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -47,6 +47,7 @@ type GPTScript struct { CacheOptions OpenAIOptions DisplayOptions + SystemToolsDir string `usage:"Directory that contains system managed tool for which GPTScript will not manage the runtime"` Color *bool `usage:"Use color in output (default true)" default:"true"` Confirm bool `usage:"Prompt before running potentially dangerous commands"` Debug bool `usage:"Enable debug logging"` @@ -146,6 +147,7 @@ func (r *GPTScript) NewGPTScriptOpts() (gptscript.Options, error) { Workspace: r.Workspace, DisablePromptServer: r.UI, DefaultModelProvider: r.DefaultModelProvider, + SystemToolsDir: r.SystemToolsDir, } if r.Confirm { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index 454debf6..3771b124 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -52,6 +52,7 @@ type Options struct { Quiet *bool Workspace string DisablePromptServer bool + SystemToolsDir string Env []string } @@ -63,6 +64,7 @@ func Complete(opts ...Options) Options { result.Runner = runner.Complete(result.Runner, opt.Runner) result.OpenAI = openai.Complete(result.OpenAI, opt.OpenAI) + result.SystemToolsDir = types.FirstSet(opt.SystemToolsDir, result.SystemToolsDir) result.CredentialContexts = opt.CredentialContexts result.Quiet = types.FirstSet(opt.Quiet, result.Quiet) result.Workspace = types.FirstSet(opt.Workspace, result.Workspace) @@ -99,7 +101,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) { } if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir()) + opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir(), opts.SystemToolsDir) } if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil { diff --git a/pkg/repos/get.go b/pkg/repos/get.go index cbc88be3..0a50ce15 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "regexp" "runtime" "strings" "sync" @@ -58,7 +59,7 @@ type Manager struct { storageDir string gitDir string runtimeDir string - systemDir string + systemDirs []string runtimes []Runtime credHelperConfig *credHelperConfig } @@ -69,14 +70,22 @@ type credHelperConfig struct { cliCfg *config.CLIConfig } -func New(cacheDir string, runtimes ...Runtime) *Manager { - root := filepath.Join(cacheDir, "repos") +func New(cacheDir, systemDir string, runtimes ...Runtime) *Manager { + var ( + systemDirs []string + root = filepath.Join(cacheDir, "repos") + ) + + if strings.TrimSpace(systemDir) != "" { + systemDirs = regexp.MustCompile("[;:,]").Split(strings.TrimSpace(systemDir), -1) + } + return &Manager{ cacheDir: cacheDir, storageDir: root, gitDir: filepath.Join(root, "git"), runtimeDir: filepath.Join(root, "runtimes"), - systemDir: filepath.Join(root, "system"), + systemDirs: systemDirs, runtimes: runtimes, } } @@ -273,8 +282,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e } func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) { - if strings.HasPrefix(tool.WorkingDir, m.systemDir) { - return tool.WorkingDir, env, nil + for _, systemDir := range m.systemDirs { + if strings.HasPrefix(tool.WorkingDir, systemDir) { + return tool.WorkingDir, env, nil + } } var isLocal bool diff --git a/pkg/repos/get_test.go b/pkg/repos/get_test.go index d59e5513..3a656dc0 100644 --- a/pkg/repos/get_test.go +++ b/pkg/repos/get_test.go @@ -19,7 +19,7 @@ var ( ) func TestManager_GetContext(t *testing.T) { - m := New(testCacheHome, &python.Runtime{ + m := New(testCacheHome, "", &python.Runtime{ Version: "3.11", }) cwd, env, err := m.GetContext(context.Background(), types.Tool{ diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index a93fb735..ea237cc4 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -30,6 +30,6 @@ var Runtimes = []repos.Runtime{ }, } -func Default(cacheDir string) engine.RuntimeManager { - return repos.New(cacheDir, Runtimes...) +func Default(cacheDir, systemDir string) engine.RuntimeManager { + return repos.New(cacheDir, systemDir, Runtimes...) } diff --git a/pkg/sdkserver/server.go b/pkg/sdkserver/server.go index 1368fd54..f0c61940 100644 --- a/pkg/sdkserver/server.go +++ b/pkg/sdkserver/server.go @@ -111,7 +111,7 @@ func run(ctx context.Context, listener net.Listener, opts Options) error { workspaceTool: opts.WorkspaceTool, client: g, events: events, - runtimeManager: runtimes.Default(opts.Options.Cache.CacheDir), // TODO - do we always want to use runtimes.Default here? + runtimeManager: runtimes.Default(opts.Options.Cache.CacheDir, opts.SystemToolsDir), waitingToConfirm: make(map[string]chan runner.AuthorizerResponse), waitingToPrompt: make(map[string]chan map[string]string), } diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index b460ce18..fa7f7683 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -196,7 +196,7 @@ func NewRunner(t *testing.T) *Runner { cacheDir, err := xdg.CacheFile("gptscript-test-cache/runtime") require.NoError(t, err) - rm := runtimes.Default(cacheDir) + rm := runtimes.Default(cacheDir, "") run, err := runner.New(c, credentials.NoopStore{}, runner.Options{ Sequential: true,