Refactor caching and error handling to use Ferret v2

Makefile

@@ -1,5 +1,5 @@
-VERSION ?= $(shell sh versions.sh worker)
-FERRET_VERSION = $(shell sh versions.sh ferret)
+VERSION ?= $(shell sh scripts/versions.sh worker)
+FERRET_VERSION = $(shell sh scripts/versions.sh ferret)
 DIR_BIN = ./bin
 
 .PHONY: build

README.md

@@ -145,6 +145,14 @@ Executes a given FQL query. The payload must have the following shape:
 }
 ```
 
+**Error response:**
+```json
+{
+  "error": "run program: Found 2 errors",
+  "details": "TypeError: runtime error\n --> anonymous:1:13\n1 | RETURN { a: @url, b: @limit }\n  |             ^^^^ parameter is required\n"
+}
+```
+
 **Example with complex data extraction:**
 ```bash
 curl -X POST http://localhost:8080/ \

go.mod

@@ -3,42 +3,34 @@ module github.com/MontFerret/worker
 go 1.25.0
 
 require (
-	github.com/MontFerret/ferret v0.18.1
+	github.com/MontFerret/ferret/v2 v2.0.0-alpha.12
 	github.com/go-waitfor/waitfor v1.1.0
 	github.com/go-waitfor/waitfor-http v1.1.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/labstack/echo/v4 v4.15.1
 	github.com/namsral/flag v1.7.4-pre
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.34.0
+	github.com/rs/zerolog v1.35.1
 	github.com/ziflex/lecho/v3 v3.10.0
 	golang.org/x/time v0.15.0
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.10.3 // indirect
-	github.com/andybalholm/cascadia v1.3.3 // indirect
-	github.com/antchfx/htmlquery v1.3.4 // indirect
-	github.com/antchfx/xpath v1.3.4 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
-	github.com/corpix/uarand v0.2.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/gorilla/css v1.0.1 // indirect
-	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/goccy/go-json v0.10.6 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
-	github.com/mafredri/cdp v0.35.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/sethgrid/pester v1.2.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/wI2L/jettison v0.7.4 // indirect
 	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
 	golang.org/x/net v0.48.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
 	golang.org/x/text v0.32.0 // indirect
 )

internal/controllers/common.go

@@ -1,7 +1,32 @@
 package controllers
 
+import (
+	"errors"
+
+	"github.com/MontFerret/ferret/v2/pkg/diagnostics"
+)
+
 type (
 	HTTPError struct {
-		Error string `json:"error"`
+		Error   string `json:"error"`
+		Details string `json:"details"`
 	}
 )
+
+func newHTTPError(err error) HTTPError {
+	if err == nil {
+		return HTTPError{}
+	}
+
+	details := err.Error()
+
+	var formattable diagnostics.Formattable
+	if errors.As(err, &formattable) {
+		details = formattable.Format()
+	}
+
+	return HTTPError{
+		Error:   err.Error(),
+		Details: details,
+	}
+}

internal/controllers/common_test.go

@@ -0,0 +1,59 @@
+package controllers
+
+import (
+	stderrors "errors"
+	"strings"
+	"testing"
+
+	pkgerrors "github.com/pkg/errors"
+
+	"github.com/MontFerret/ferret/v2/pkg/diagnostics"
+	"github.com/MontFerret/ferret/v2/pkg/source"
+)
+
+func TestNewHTTPErrorFormatsWrappedDiagnostics(t *testing.T) {
+	src := source.New("query.fql", "RETURN @missing")
+	diag := &diagnostics.Diagnostic{
+		Kind:    diagnostics.TypeError,
+		Message: "missing query parameter",
+		Source:  src,
+		Spans: []diagnostics.ErrorSpan{
+			diagnostics.NewMainErrorSpan(source.Span{Start: 7, End: 15}, "parameter is required"),
+		},
+	}
+	diagSet := diagnostics.NewDiagnosticsOf[*diagnostics.Diagnostic]([]*diagnostics.Diagnostic{diag})
+
+	got := newHTTPError(pkgerrors.Wrap(diagSet, "compile query"))
+
+	if got.Error != "compile query: Found 1 errors" {
+		t.Fatalf("unexpected error: %q", got.Error)
+	}
+
+	if got.Details != diagSet.Format() {
+		t.Fatalf("expected formatted diagnostics, got:\n%s", got.Details)
+	}
+
+	for _, want := range []string{
+		"TypeError: missing query parameter",
+		"--> query.fql:1:8",
+		"parameter is required",
+	} {
+		if !strings.Contains(got.Details, want) {
+			t.Fatalf("expected details to contain %q, got:\n%s", want, got.Details)
+		}
+	}
+}
+
+func TestNewHTTPErrorFallsBackToPlainError(t *testing.T) {
+	err := stderrors.New("invalid request body")
+
+	got := newHTTPError(err)
+
+	if got.Error != err.Error() {
+		t.Fatalf("unexpected error: %q", got.Error)
+	}
+
+	if got.Details != err.Error() {
+		t.Fatalf("unexpected details: %q", got.Details)
+	}
+}

internal/controllers/worker.go

@@ -43,7 +43,7 @@ func (c *Worker) runScript(ctx echo.Context) error {
 
 		return ctx.JSON(
 			http.StatusBadRequest,
-			HTTPError{err.Error()},
+			newHTTPError(err),
 		)
 	}
 
@@ -56,7 +56,7 @@ func (c *Worker) runScript(ctx echo.Context) error {
 
 		return ctx.JSON(
 			http.StatusBadRequest,
-			HTTPError{err.Error()},
+			newHTTPError(err),
 		)
 	}
 

internal/controllers/worker_test.go

@@ -0,0 +1,70 @@
+package controllers
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/labstack/echo/v4"
+
+	workerpkg "github.com/MontFerret/worker/pkg/worker"
+)
+
+func TestWorkerRunScriptReturnsDiagnosticDetails(t *testing.T) {
+	wkr, err := workerpkg.New()
+	if err != nil {
+		t.Fatalf("new worker: %v", err)
+	}
+
+	controller, err := NewWorker(wkr)
+	if err != nil {
+		t.Fatalf("new worker controller: %v", err)
+	}
+
+	e := echo.New()
+	controller.Use(e)
+
+	req := httptest.NewRequest(
+		http.MethodPost,
+		"/",
+		strings.NewReader(`{"text":"RETURN { a: @foo, b: @bar }"}`),
+	)
+	req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	rec := httptest.NewRecorder()
+
+	e.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("expected status %d, got %d: %s", http.StatusBadRequest, rec.Code, rec.Body.String())
+	}
+
+	var payload HTTPError
+	if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+
+	if !strings.Contains(payload.Error, "run program: Found") {
+		t.Fatalf("expected wrapped execution error, got %q", payload.Error)
+	}
+
+	if payload.Details == "" {
+		t.Fatal("expected details to be populated")
+	}
+
+	if payload.Details == payload.Error {
+		t.Fatalf("expected formatted diagnostics, got only plain error: %q", payload.Details)
+	}
+
+	for _, want := range []string{
+		"anonymous",
+		"RETURN { a: @foo, b: @bar }",
+		"@foo",
+		"@bar",
+	} {
+		if !strings.Contains(payload.Details, want) {
+			t.Fatalf("expected details to contain %q, got:\n%s", want, payload.Details)
+		}
+	}
+}

internal/storage/cache.go

@@ -1,21 +1,20 @@
 package storage
 
 import (
+	"github.com/MontFerret/ferret/v2"
 	lru "github.com/hashicorp/golang-lru/v2"
 
-	"github.com/MontFerret/ferret/pkg/runtime"
-
 	"github.com/MontFerret/worker/pkg/caching"
 )
 
 type InMemoryCache struct {
-	store *lru.Cache[string, *runtime.Program]
+	store *lru.Cache[string, *ferret.Plan]
 }
 
-func NewCache(opts ...caching.Option) (caching.Cache[*runtime.Program], error) {
+func NewCache(opts ...caching.Option) (caching.Cache[*ferret.Plan], error) {
 	options := caching.NewOptions(opts...)
 
-	store, err := lru.New[string, *runtime.Program](int(options.Size))
+	store, err := lru.New[string, *ferret.Plan](int(options.Size))
 
 	if err != nil {
 		return nil, err
@@ -24,11 +23,11 @@ func NewCache(opts ...caching.Option) (caching.Cache[*runtime.Program], error) {
 	return &InMemoryCache{store}, nil
 }
 
-func (cache *InMemoryCache) Set(key string, value *runtime.Program) {
+func (cache *InMemoryCache) Set(key string, value *ferret.Plan) {
 	cache.store.Add(key, value)
 }
 
-func (cache *InMemoryCache) Get(key string) (*runtime.Program, bool) {
+func (cache *InMemoryCache) Get(key string) (*ferret.Plan, bool) {
 	return cache.store.Get(key)
 }
 

pkg/worker/models.go

@@ -9,6 +9,7 @@ type (
 
 	// Result is the result of Query.
 	Result struct {
-		Raw []byte
+		ContentType string
+		Raw         []byte
 	}
 )

pkg/worker/options.go

@@ -3,8 +3,8 @@ package worker
 import (
 	"fmt"
 
-	"github.com/MontFerret/ferret/pkg/runtime"
-	"github.com/MontFerret/ferret/pkg/runtime/core"
+	"github.com/MontFerret/ferret/v2"
+	"github.com/MontFerret/ferret/v2/pkg/runtime"
 	"github.com/MontFerret/worker/pkg/caching"
 )
 
@@ -16,10 +16,9 @@ type (
 	}
 
 	Options struct {
-		functions []core.Functions
-		noStdlib  bool
-		cdp       CDPSettings
-		cache     caching.Cache[*runtime.Program]
+		engine []ferret.Option
+		cdp    CDPSettings
+		cache  caching.Cache[*ferret.Plan]
 	}
 
 	Option func(opts *Options)
@@ -35,23 +34,23 @@ func (s CDPSettings) VersionURL() string {
 
 func newOptions() *Options {
 	return &Options{
-		functions: make([]core.Functions, 0, 5),
+		engine: []ferret.Option{},
 		cdp: CDPSettings{
 			Host: "127.0.0.1",
 			Port: 9222,
 		},
 	}
 }
 
-func WithFunctions(functions core.Functions) Option {
+func WithFunctions(functions *runtime.Functions) Option {
 	return func(opts *Options) {
-		opts.functions = append(opts.functions, functions)
+		opts.engine = append(opts.engine, ferret.WithFunctions(functions))
 	}
 }
 
 func WithoutStdlib() Option {
 	return func(opts *Options) {
-		opts.noStdlib = true
+		opts.engine = append(opts.engine, ferret.WithoutStdlib())
 	}
 }
 
@@ -61,7 +60,7 @@ func WithCustomCDP(cdp CDPSettings) Option {
 	}
 }
 
-func WithCache(cache caching.Cache[*runtime.Program]) Option {
+func WithCache(cache caching.Cache[*ferret.Plan]) Option {
 	return func(opts *Options) {
 		opts.cache = cache
 	}