From d46ec1c2f77a71614a85d9322c51d0f15a6c4344 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 22 Oct 2024 15:09:10 -0400 Subject: [PATCH 1/5] add Add Elements tool to dataset sdk Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 85 +++++++++++++++++++++++++++++++++++---- pkg/sdkserver/routes.go | 1 + 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index a65566a4..e252c5bf 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -12,14 +12,11 @@ import ( type datasetRequest struct { Input string `json:"input"` - Workspace string `json:"workspace"` DatasetToolRepo string `json:"datasetToolRepo"` } func (r datasetRequest) validate(requireInput bool) error { - if r.Workspace == "" { - return fmt.Errorf("workspace is required") - } else if requireInput && r.Input == "" { + if requireInput && r.Input == "" { return fmt.Errorf("input is required") } return nil @@ -27,10 +24,9 @@ func (r datasetRequest) validate(requireInput bool) error { func (r datasetRequest) opts(o gptscript.Options) gptscript.Options { opts := gptscript.Options{ - Cache: o.Cache, - Monitor: o.Monitor, - Runner: o.Runner, - Workspace: r.Workspace, + Cache: o.Cache, + Monitor: o.Monitor, + Runner: o.Runner, } return opts } @@ -209,6 +205,79 @@ func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { writeResponse(logger, w, map[string]any{"stdout": result}) } +type addDatasetElementsArgs struct { + DatasetID string `json:"datasetID"` + Elements []struct { + Name string `json:"name"` + Description string `json:"description"` + Contents string `json:"contents"` + } +} + +func (a addDatasetElementsArgs) validate() error { + if a.DatasetID == "" { + return fmt.Errorf("datasetID is required") + } + if len(a.Elements) == 0 { + return fmt.Errorf("elements is required") + } + return nil +} + +func (s *server) addDatasetElements(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(true); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + var args addDatasetElementsArgs + if err := json.Unmarshal([]byte(req.Input), &args); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to unmarshal input: %w", err)) + return + } + + if err := args.validate(); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + prg, err := loader.Program(r.Context(), req.getToolRepo(), "Add Elements", loader.Options{ + Cache: g.Cache, + }) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + elementsJSON, err := json.Marshal(args.Elements) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to marshal elements: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, fmt.Sprintf(`{"datasetID":%q, elements:%s}`, args.DatasetID, elementsJSON)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, map[string]any{"stdout": result}) +} + type listDatasetElementsArgs struct { DatasetID string `json:"datasetID"` } diff --git a/pkg/sdkserver/routes.go b/pkg/sdkserver/routes.go index 8afdb8a4..713f74fe 100644 --- a/pkg/sdkserver/routes.go +++ b/pkg/sdkserver/routes.go @@ -73,6 +73,7 @@ func (s *server) addRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /datasets/list-elements", s.listDatasetElements) mux.HandleFunc("POST /datasets/get-element", s.getDatasetElement) mux.HandleFunc("POST /datasets/add-element", s.addDatasetElement) + mux.HandleFunc("POST /datasets/add-elements", s.addDatasetElements) mux.HandleFunc("POST /workspaces/create", s.createWorkspace) mux.HandleFunc("POST /workspaces/delete", s.deleteWorkspace) From 1ebbecc7a3b4b1dd292e39f067eaec3185ad000b Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 22 Oct 2024 15:10:27 -0400 Subject: [PATCH 2/5] update default dataset tool repo Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index e252c5bf..822f9bd2 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -35,7 +35,7 @@ func (r datasetRequest) getToolRepo() string { if r.DatasetToolRepo != "" { return r.DatasetToolRepo } - return "github.com/gptscript-ai/datasets" + return "github.com/otto8-ai/datasets" } func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { From 9e4bf1e9efec00336356de100c1117e66976e687 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 22 Oct 2024 20:49:25 -0400 Subject: [PATCH 3/5] add workspaceID arg Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 822f9bd2..21fb162b 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -12,11 +12,14 @@ import ( type datasetRequest struct { Input string `json:"input"` + WorkspaceID string `json:"workspaceID"` DatasetToolRepo string `json:"datasetToolRepo"` } func (r datasetRequest) validate(requireInput bool) error { - if requireInput && r.Input == "" { + if r.WorkspaceID == "" { + return fmt.Errorf("workspaceID is required") + } else if requireInput && r.Input == "" { return fmt.Errorf("input is required") } return nil @@ -24,9 +27,10 @@ func (r datasetRequest) validate(requireInput bool) error { func (r datasetRequest) opts(o gptscript.Options) gptscript.Options { opts := gptscript.Options{ - Cache: o.Cache, - Monitor: o.Monitor, - Runner: o.Runner, + Cache: o.Cache, + Monitor: o.Monitor, + Runner: o.Runner, + Workspace: r.WorkspaceID, } return opts } @@ -269,7 +273,7 @@ func (s *server) addDatasetElements(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, fmt.Sprintf(`{"datasetID":%q, elements:%s}`, args.DatasetID, elementsJSON)) + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, fmt.Sprintf(`{"datasetID":%q, "elements":%q}`, args.DatasetID, string(elementsJSON))) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return From f402c56921997db96d5361910b22d68ecbd65b7e Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 23 Oct 2024 11:44:15 -0400 Subject: [PATCH 4/5] add env to the request Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 21fb162b..b5f1353f 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -11,9 +11,10 @@ import ( ) type datasetRequest struct { - Input string `json:"input"` - WorkspaceID string `json:"workspaceID"` - DatasetToolRepo string `json:"datasetToolRepo"` + Input string `json:"input"` + WorkspaceID string `json:"workspaceID"` + DatasetToolRepo string `json:"datasetToolRepo"` + Env []string `json:"env"` } func (r datasetRequest) validate(requireInput bool) error { @@ -71,7 +72,7 @@ func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + result, err := g.Run(r.Context(), prg, req.Env, req.Input) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return @@ -132,7 +133,7 @@ func (s *server) createDataset(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + result, err := g.Run(r.Context(), prg, req.Env, req.Input) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return @@ -200,7 +201,7 @@ func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + result, err := g.Run(r.Context(), prg, req.Env, req.Input) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return @@ -273,7 +274,7 @@ func (s *server) addDatasetElements(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, fmt.Sprintf(`{"datasetID":%q, "elements":%q}`, args.DatasetID, string(elementsJSON))) + result, err := g.Run(r.Context(), prg, req.Env, fmt.Sprintf(`{"datasetID":%q, "elements":%q}`, args.DatasetID, string(elementsJSON))) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return @@ -332,7 +333,7 @@ func (s *server) listDatasetElements(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + result, err := g.Run(r.Context(), prg, req.Env, req.Input) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return @@ -395,7 +396,7 @@ func (s *server) getDatasetElement(w http.ResponseWriter, r *http.Request) { return } - result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + result, err := g.Run(r.Context(), prg, req.Env, req.Input) if err != nil { writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) return From 92f8f6a3549c723e3e2aac66a0221caaa02ba9a8 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 23 Oct 2024 14:33:58 -0400 Subject: [PATCH 5/5] require env Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index b5f1353f..bfef595a 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -22,6 +22,8 @@ func (r datasetRequest) validate(requireInput bool) error { return fmt.Errorf("workspaceID is required") } else if requireInput && r.Input == "" { return fmt.Errorf("input is required") + } else if len(r.Env) == 0 { + return fmt.Errorf("env is required") } return nil }