Add support for flaresolverr

.vscode/settings.json

@@ -20,6 +20,7 @@
   "cSpell.words": [
     "ahobsonsayers",
     "delisted",
+    "flaresolverr",
     "godotenv",
     "httpmock",
     "nolint",

client.go

@@ -104,7 +104,7 @@ func (c *Client) FetchTicketListingsByFeedUrl(
 
 	bodyBytes, err := io.ReadAll(response.Body)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed ready response body: %w", err)
 	}
 
 	return UnmarshalTwicketsFeedJson(bodyBytes)
@@ -196,13 +196,23 @@ func processFeedListings(
 	return processedListings, false
 }
 
+type ClientOpt func(*req.Client) error
+
 // NewClient creates a new Twickets client
-func NewClient(apiKey string) (*Client, error) {
+func NewClient(apiKey string, opts ...ClientOpt) (*Client, error) {
 	if apiKey == "" {
 		return nil, errors.New("api key must be set")
 	}
 
-	client := req.C().ImpersonateChrome()
+	client := req.C()
+	client = client.ImpersonateChrome()
+	for _, opt := range opts {
+		err := opt(client)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &Client{
 		client: client,
 		apiKey: apiKey,

client_test.go

@@ -59,7 +59,10 @@ func TestFetchListingsReal(t *testing.T) {
 	twicketsAPIKey := os.Getenv("TWICKETS_API_KEY")
 	require.NotEmpty(t, twicketsAPIKey, "TWICKETS_API_KEY is not set")
 
-	twicketsClient, err := twigots.NewClient(twicketsAPIKey)
+	twicketsClient, err := twigots.NewClient(
+		twicketsAPIKey,
+		twigots.WithFlareSolverr("http://0.0.0.0:8191"),
+	)
 	require.NoError(t, err)
 
 	// Fetch ticket listings

compose.yaml

@@ -0,0 +1,11 @@
+services:
+  flaresolverr:
+    container_name: flaresolverr
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    restart: unless-stopped
+    ports:
+      - 8191:8191
+    environment:
+      - LOG_LEVEL=info
+      - LOG_HTML=true
+      - TZ=Europe/London

flaresolverr.go

@@ -0,0 +1,145 @@
+package twigots
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/imroc/req/v3"
+	"github.com/tidwall/gjson"
+)
+
+func WithFlareSolverr(flareSolverrUrl string) ClientOpt {
+	return func(client *req.Client) error {
+		err := ValidateURL(flareSolverrUrl)
+		if err != nil {
+			return fmt.Errorf("invalid flaresolverr url: %w", err)
+		}
+
+		// Ensure url path ends with /v1
+		// TODO this could be done better
+		flareSolverrUrl := strings.TrimSuffix(flareSolverrUrl, "/")
+		flareSolverrUrl = strings.TrimSuffix(flareSolverrUrl, "/v1")
+		flareSolverrUrl = fmt.Sprintf("%s/v1", flareSolverrUrl)
+
+		// Apply middleware
+		flareSolverrMiddleware := getFlareSolverrMiddleware(flareSolverrUrl)
+		client.WrapRoundTripFunc(flareSolverrMiddleware)
+
+		return nil
+	}
+}
+
+func getFlareSolverrMiddleware(flareSolverrUrl string) req.RoundTripWrapperFunc {
+	return func(rt req.RoundTripper) req.RoundTripFunc {
+		return func(request *req.Request) (*req.Response, error) {
+			// Before request
+			err := transformToFlareSolverrRequest(request, flareSolverrUrl)
+			if err != nil {
+				return nil, fmt.Errorf(
+					"failed to transform standard request to flaresolverr request: %w",
+					err,
+				)
+			}
+
+			// Do request
+			response, err := rt.RoundTrip(request)
+			if err != nil {
+				return response, err
+			}
+
+			// After request
+			err = transformFromFlareSolverrResponse(response)
+			if err != nil {
+				return response, fmt.Errorf(
+					"failed to transform flaresolverr response to standard response: %w",
+					err,
+				)
+			}
+
+			return response, nil
+		}
+	}
+}
+
+// transformToFlareSolverrRequest transforms a standard response to a flaresolverr request.
+// The passed requests in modified in place
+func transformToFlareSolverrRequest(request *req.Request, flareSolverrUrl string) error {
+	if request.Method != http.MethodGet {
+		return nil
+	}
+
+	parsedFlareSolverrUrl, err := url.Parse(flareSolverrUrl)
+	if err != nil {
+		return fmt.Errorf("failed to parse flaresolverr url: %w", err)
+	}
+
+	twicketsRawUrl := request.RawURL
+
+	// Update request to be made to flaresolverr
+	request.Method = http.MethodPost
+	request.RawURL = flareSolverrUrl
+	request.URL = parsedFlareSolverrUrl
+	request.SetBodyJsonMarshal(
+		map[string]any{
+			"cmd":        "request.get",
+			"url":        twicketsRawUrl,
+			"maxTimeout": "5000",
+		})
+
+	return nil // return nil if it is success
+}
+
+// transformFromFlareSolverrResponse transforms a flaresolverr response to a standard response.
+// The passed response in modified in place
+func transformFromFlareSolverrResponse(response *req.Response) error {
+	if response.Err != nil { // you can skip if error occurs.
+		return nil
+	}
+
+	proxyBodyBytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+
+	// Extract json response from flaresolvarr response
+	bodyBytesResult := gjson.GetBytes(proxyBodyBytes, "solution.response")
+	bodyString := bodyBytesResult.String()
+	bodyReader := strings.NewReader(bodyString)
+	bodyDoc, err := goquery.NewDocumentFromReader(bodyReader)
+	if err != nil {
+		return fmt.Errorf("failed to parse response body: %w", err)
+	}
+	bodyJson := bodyDoc.Find("pre").Text()
+
+	response.Body = io.NopCloser(strings.NewReader(bodyJson))
+
+	return nil // return nil if it is success
+}
+
+// ValidateURL checks if a url string  is a valid.
+func ValidateURL(urlString string) error {
+	if urlString == "" {
+		return errors.New("url is not set ")
+	}
+
+	parsedURL, err := url.Parse(urlString)
+	if err != nil {
+		return fmt.Errorf("url format invalid: %w", err)
+	}
+
+	if parsedURL.Host == "" {
+		return errors.New("url hostname missing ")
+	}
+
+	scheme := strings.ToLower(parsedURL.Scheme)
+	if scheme != "http" && scheme == "https" {
+		return fmt.Errorf("url scheme unsupported: %v", parsedURL.Scheme)
+	}
+
+	return nil
+}

go.mod

@@ -8,6 +8,7 @@ tool (
 )
 
 require (
+	github.com/PuerkitoBio/goquery v1.10.3
 	github.com/ahobsonsayers/utilopia v0.2.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/hbollon/go-edlib v1.6.0
@@ -17,6 +18,7 @@ require (
 	github.com/k3a/html2text v1.2.1
 	github.com/orsinium-labs/enum v1.4.0
 	github.com/stretchr/testify v1.11.1
+	github.com/tidwall/gjson v1.18.0
 	golang.org/x/text v0.30.0
 )
 
@@ -40,12 +42,15 @@ require (
 	github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect
 	github.com/alecthomas/chroma/v2 v2.20.0 // indirect
 	github.com/alecthomas/go-check-sumtype v0.3.1 // indirect
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.6 // indirect
 	github.com/alexkohler/prealloc v1.0.0 // indirect
 	github.com/alfatraining/structtag v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/alingse/nilnesserr v0.2.0 // indirect
 	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect
 	github.com/ashanbrown/makezero/v2 v2.1.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
@@ -118,6 +123,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hexops/gotextdiff v1.0.3 // indirect
+	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/icholy/digest v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jgautheron/goconst v1.8.2 // indirect
@@ -149,6 +155,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mgechev/revive v1.12.0 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moricho/tparallel v0.3.2 // indirect
@@ -183,6 +190,7 @@ require (
 	github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
 	github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect
 	github.com/securego/gosec/v2 v2.22.10 // indirect
+	github.com/segmentio/golines v0.12.2 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sonatard/noctx v0.4.0 // indirect
@@ -198,6 +206,8 @@ require (
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/tetafro/godot v1.5.4 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect
 	github.com/timonwong/loggercheck v0.11.0 // indirect
 	github.com/tomarrell/wrapcheck/v2 v2.11.0 // indirect
@@ -206,6 +216,7 @@ require (
 	github.com/ultraware/whitespace v0.2.0 // indirect
 	github.com/uudashr/gocognit v1.2.0 // indirect
 	github.com/uudashr/iface v1.4.1 // indirect
+	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
 	github.com/xen0n/gosmopolitan v1.3.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect
@@ -226,12 +237,18 @@ require (
 	golang.org/x/net v0.46.0 // indirect
 	golang.org/x/sync v0.18.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/term v0.36.0 // indirect
 	golang.org/x/tools v0.38.0 // indirect
 	google.golang.org/protobuf v1.36.8 // indirect
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
+	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.6.1 // indirect
 	mvdan.cc/gofumpt v0.9.2 // indirect
 	mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect
 )
+
+tool github.com/segmentio/golines