Merge pull request #154 from ibuildthecloud/windows

Windows path fixes

Author: ibuildthecloud

pkg/engine/cmd.go

@@ -8,11 +8,13 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"runtime"
 	"sort"
 	"strings"
 	"sync/atomic"
 
 	"github.com/google/shlex"
+	"github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/gptscript-ai/gptscript/pkg/version"
 )
@@ -103,6 +105,7 @@ func envAsMapAndDeDup(env []string) (sortedEnv []string, _ map[string]string) {
 
 var ignoreENV = map[string]struct{}{
 	"PATH":               {},
+	"Path":               {},
 	"GPTSCRIPT_TOOL_DIR": {},
 }
 
@@ -146,8 +149,8 @@ func appendInputAsEnv(env []string, input string) []string {
 }
 
 func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.Tool, input string) (*exec.Cmd, func(), error) {
-	env := append(e.Env[:], extraEnv...)
-	env = appendInputAsEnv(env, input)
+	envvars := append(e.Env[:], extraEnv...)
+	envvars = appendInputAsEnv(envvars, input)
 
 	interpreter, rest, _ := strings.Cut(tool.Instructions, "\n")
 	interpreter = strings.TrimSpace(interpreter)[2:]
@@ -157,18 +160,22 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 		return nil, nil, err
 	}
 
-	env, err = e.getRuntimeEnv(ctx, tool, args, env)
+	envvars, err = e.getRuntimeEnv(ctx, tool, args, envvars)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	env, envMap := envAsMapAndDeDup(env)
+	envvars, envMap := envAsMapAndDeDup(envvars)
 	for i, arg := range args {
 		args[i] = os.Expand(arg, func(s string) string {
 			return envMap[s]
 		})
 	}
 
+	if runtime.GOOS == "windows" && (args[0] == "/usr/bin/env" || args[0] == "/bin/env") {
+		args = args[1:]
+	}
+
 	var (
 		cmdArgs = args[1:]
 		stop    = func() {}
@@ -192,7 +199,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 		cmdArgs = append(cmdArgs, f.Name())
 	}
 
-	cmd := exec.CommandContext(ctx, args[0], cmdArgs...)
-	cmd.Env = env
+	cmd := exec.CommandContext(ctx, env.Lookup(envvars, args[0]), cmdArgs...)
+	cmd.Env = envvars
 	return cmd, stop, nil
 }

pkg/env/env.go

@@ -0,0 +1,73 @@
+package env
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func execEquals(bin, check string) bool {
+	return bin == check ||
+		bin == check+".exe"
+}
+
+func Matches(cmd []string, bin string) bool {
+	switch len(cmd) {
+	case 0:
+		return false
+	case 1:
+		return execEquals(cmd[0], bin)
+	}
+	if cmd[0] == bin {
+		return true
+	}
+	if cmd[0] == "/usr/bin/env" || cmd[0] == "/bin/env" {
+		return execEquals(cmd[1], bin)
+	}
+	return false
+}
+
+func AppendPath(env []string, binPath string) []string {
+	var newEnv []string
+	for _, path := range env {
+		v, ok := strings.CutPrefix(path, "PATH=")
+		if ok {
+			newEnv = append(newEnv, fmt.Sprintf("PATH=%s%s%s",
+				binPath, string(os.PathListSeparator), v))
+		}
+	}
+	return newEnv
+}
+
+// Lookup will try to find bin in the PATH in env. It will refer to PATHEXT for Windows support.
+// If bin can not be resolved to anything the original bin string is returned.
+func Lookup(env []string, bin string) string {
+	for _, env := range env {
+		for _, prefix := range []string{"PATH=", "Path="} {
+			suffix, ok := strings.CutPrefix(env, prefix)
+			if !ok {
+				continue
+			}
+			for _, path := range strings.Split(suffix, string(os.PathListSeparator)) {
+				testPath := filepath.Join(path, bin)
+
+				if stat, err := os.Stat(testPath); err == nil && !stat.IsDir() {
+					return testPath
+				}
+
+				for _, ext := range strings.Split(os.Getenv("PATHEXT"), string(os.PathListSeparator)) {
+					if ext == "" {
+						continue
+					}
+
+					if stat, err := os.Stat(testPath + ext); err == nil && !stat.IsDir() {
+						return testPath + ext
+					}
+				}
+			}
+		}
+	}
+
+	return bin
+}

pkg/loader/url.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	url2 "net/url"
-	"path/filepath"
+	"path"
 	"strings"
 
 	"github.com/gptscript-ai/gptscript/pkg/types"
@@ -32,9 +32,9 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err
 
 	if base.Repo != nil {
 		newRepo := *base.Repo
-		newPath := filepath.Join(newRepo.Path, name)
-		newRepo.Path = filepath.Dir(newPath)
-		newRepo.Name = filepath.Base(newPath)
+		newPath := path.Join(newRepo.Path, name)
+		newRepo.Path = path.Dir(newPath)
+		newRepo.Name = path.Base(newPath)
 		repo = &newRepo
 	}
 
@@ -61,10 +61,10 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err
 	}
 
 	pathURL := *parsed
-	pathURL.Path = filepath.Dir(parsed.Path)
-	path := pathURL.String()
-	name = filepath.Base(parsed.Path)
-	url = path + "/" + name
+	pathURL.Path = path.Dir(parsed.Path)
+	pathString := pathURL.String()
+	name = path.Base(parsed.Path)
+	url = pathString + "/" + name
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
@@ -83,7 +83,7 @@ func loadURL(ctx context.Context, base *source, name string) (*source, bool, err
 	return &source{
 		Content:  resp.Body,
 		Remote:   true,
-		Path:     path,
+		Path:     pathString,
 		Name:     name,
 		Location: url,
 		Repo:     repo,

pkg/repos/download/extract.go

@@ -24,23 +24,43 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error {
 		return fmt.Errorf("mkdir %s: %w", targetDir, err)
 	}
 
+	tmpFile, err := os.CreateTemp("", "gptscript-download")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
 	resp, err := http.Get(downloadURL)
 	if err != nil {
 		return err
 	}
 	defer resp.Body.Close()
 
-	// NOTE: Because I'm validating the hash at the same time as extracting this isn't actually secure.
-	// Security is still assumed the source is trusted.  Which is bad and should be changed.
 	digester := sha256.New()
 	input := io.TeeReader(resp.Body, digester)
 
+	if _, err = io.Copy(tmpFile, input); err != nil {
+		return err
+	}
+
+	resultDigest := digester.Sum(nil)
+	resultDigestString := hex.EncodeToString(resultDigest[:])
+
+	if resultDigestString != digest {
+		return fmt.Errorf("downloaded %s and expected digest %s but got %s", downloadURL, digest, resultDigestString)
+	}
+
 	parsedURL, err := url.Parse(downloadURL)
 	if err != nil {
 		return err
 	}
 
-	format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), input)
+	if _, err := tmpFile.Seek(0, 0); err != nil {
+		return err
+	}
+
+	format, input, err := archiver.Identify(filepath.Base(parsedURL.Path), tmpFile)
 	if err != nil {
 		return err
 	}
@@ -90,12 +110,5 @@ func Extract(ctx context.Context, downloadURL, digest, targetDir string) error {
 		return err
 	}
 
-	resultDigest := digester.Sum(nil)
-	resultDigestString := hex.EncodeToString(resultDigest[:])
-
-	if resultDigestString != digest {
-		return fmt.Errorf("downloaded %s and expected digest %s but got %s", downloadURL, digest, resultDigestString)
-	}
-
 	return nil
 }

pkg/repos/get.go

@@ -93,6 +93,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 		return "", nil, err
 	}
 
+	if err := out.Close(); err != nil {
+		return "", nil, err
+	}
+
 	return targetFinal, append(env, newEnv...), os.Rename(doneFile+".tmp", doneFile)
 }
 

pkg/repos/runtimes/env/env.go

@@ -14,9 +14,9 @@ import (
 	"strings"
 
 	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
-	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env"
 )
 
 //go:embed digests.txt

pkg/repos/runtimes/golang/golang.go

@@ -14,9 +14,9 @@ import (
 	"strings"
 
 	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
-	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env"
 )
 
 //go:embed SHASUMS256.txt.asc
@@ -84,7 +84,7 @@ func arch() string {
 
 func (r *Runtime) getReleaseAndDigest() (string, string, error) {
 	scanner := bufio.NewScanner(bytes.NewReader(releasesData))
-	key := osName() + "-" + arch()
+	key := "-" + osName() + "-" + arch()
 	for scanner.Scan() {
 		line := scanner.Text()
 		if strings.Contains(line, "node-v"+r.Version) && strings.Contains(line, key) {
@@ -116,6 +116,11 @@ func (r *Runtime) binDir(rel string) (string, error) {
 
 	for _, entry := range entries {
 		if entry.IsDir() {
+			if _, err := os.Stat(filepath.Join(rel, entry.Name(), "node.exe")); err == nil {
+				return filepath.Join(rel, entry.Name()), nil
+			} else if !errors.Is(err, fs.ErrNotExist) {
+				return "", err
+			}
 			return filepath.Join(rel, entry.Name(), "bin"), nil
 		}
 	}

pkg/repos/runtimes/node/node.go

@@ -13,9 +13,9 @@ import (
 	"runtime"
 
 	"github.com/gptscript-ai/gptscript/pkg/debugcmd"
+	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
-	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/env"
 )
 
 //go:embed python.json