From 055ada9bbaac9c23da0d29ac840eeac1fd8dcdab Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Fri, 18 Jul 2025 21:46:02 +0100 Subject: [PATCH 1/3] feat: add proxy struct --- proxy.go | 91 ++++++++++++++++++++ proxy_test.go | 226 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 proxy.go create mode 100644 proxy_test.go diff --git a/proxy.go b/proxy.go new file mode 100644 index 0000000..7092aa1 --- /dev/null +++ b/proxy.go @@ -0,0 +1,91 @@ +package twigots + +import ( + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" +) + +type Proxy struct { + Host string + Port int + User string + Password string +} + +// NewProxy creates a new Proxy instance with the given host, port, user, and password. +func NewProxy(host string, port int, user, password string) *Proxy { + return &Proxy{ + Host: host, + Port: port, + User: user, + Password: password, + } +} + +// GenerateProxyList creates a list of Proxy instances from a list of proxy URLs. +func GenerateProxyList(proxyHosts []string, user, password string) ([]Proxy, error) { + var proxyList []Proxy + for _, p := range proxyHosts { + parsedUrl, err := url.Parse(p) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy URL '%s': %w", p, err) + } + if parsedUrl.Scheme != "socks5" { + return nil, fmt.Errorf("unsupported proxy scheme for '%s': %s", p, parsedUrl.Scheme) + } + host, port, err := net.SplitHostPort(parsedUrl.Host) + if err != nil { + return nil, fmt.Errorf("failed to split host and port for '%s': %w", p, err) + } + portNum, err := strconv.Atoi(port) + if err != nil { + return nil, fmt.Errorf("invalid port number for '%s': %w", p, err) + } + proxyList = append(proxyList, Proxy{ + Host: host, + Port: portNum, + User: user, + Password: password, + }) + } + return proxyList, nil +} + +// String returns the proxy URL as a string. +func (p *Proxy) String() (string, error) { + parsedUrl, err := p.URL() + if err != nil { + return "", fmt.Errorf("failed to get proxy URL: %w", err) + } + return parsedUrl.String(), nil +} + +// URL returns the proxy URL as a *url.URL instance. +func (p *Proxy) URL() (*url.URL, error) { + if p.Host == "" || p.Port <= 0 { + return nil, errors.New("host and port must be set for proxy URL") + } + + var proxyUrl string + + if strings.TrimSpace(p.User) == "" || strings.TrimSpace(p.Password) == "" { + proxyUrl = fmt.Sprintf("socks5://%s:%d", p.Host, p.Port) + } else { + proxyUrl = fmt.Sprintf( + "socks5://%s:%s@%s:%d", + p.User, + p.Password, + p.Host, + p.Port, + ) + } + parsedUrl, err := url.Parse(proxyUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy URL: %w", err) + } + return parsedUrl, nil +} diff --git a/proxy_test.go b/proxy_test.go new file mode 100644 index 0000000..ca0592a --- /dev/null +++ b/proxy_test.go @@ -0,0 +1,226 @@ +package twigots + +import ( + "net/url" + "reflect" + "testing" +) + +func TestNewProxy(t *testing.T) { + type args struct { + host string + port int + user string + password string + } + tests := []struct { + name string + args args + want *Proxy + }{ + { + name: "Basic Proxy", + args: args{ + host: "localhost", + port: 8080, + user: "user", + password: "password", + }, + want: &Proxy{ + Host: "localhost", + Port: 8080, + User: "user", + Password: "password", + }, + }, + { + name: "Proxy without user and password", + args: args{ + host: "localhost", + port: 8080, + }, + want: &Proxy{ + Host: "localhost", + Port: 8080, + User: "", + Password: "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewProxy( + tt.args.host, + tt.args.port, + tt.args.user, + tt.args.password, + ); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewProxy() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestProxy_String(t *testing.T) { + type fields struct { + Host string + Port int + User string + Password string + } + tests := []struct { + name string + fields fields + want string + wantErr bool + }{ + { + name: "Valid Proxy with User and Password", + fields: fields{ + Host: "localhost", + Port: 8080, + User: "user", + Password: "password", + }, + want: "socks5://user:password@localhost:8080", + }, + { + name: "Valid Proxy without User and Password", + fields: fields{ + Host: "localhost", + Port: 8080, + User: "", + Password: "", + }, + want: "socks5://localhost:8080", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &Proxy{ + Host: tt.fields.Host, + Port: tt.fields.Port, + User: tt.fields.User, + Password: tt.fields.Password, + } + got, err := p.String() + if (err != nil) != tt.wantErr { + t.Errorf("Proxy.String() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Proxy.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestProxy_URL(t *testing.T) { + type fields struct { + Host string + Port int + User string + Password string + } + tests := []struct { + name string + fields fields + want *url.URL + wantErr bool + }{ + { + name: "Valid Proxy with User and Password", + fields: fields{ + Host: "localhost", + Port: 8080, + User: "user", + Password: "password", + }, + want: &url.URL{ + Scheme: "socks5", + Host: "localhost:8080", + User: url.UserPassword("user", "password"), + }, + }, + { + name: "Valid Proxy without User and Password", + fields: fields{ + Host: "localhost", + Port: 8080, + User: "", + Password: "", + }, + want: &url.URL{ + Scheme: "socks5", + Host: "localhost:8080", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &Proxy{ + Host: tt.fields.Host, + Port: tt.fields.Port, + User: tt.fields.User, + Password: tt.fields.Password, + } + got, err := p.URL() + if (err != nil) != tt.wantErr { + t.Errorf("Proxy.URL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Proxy.URL() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGenerateProxyList(t *testing.T) { + type args struct { + proxyHosts []string + user string + password string + } + tests := []struct { + name string + args args + want []Proxy + wantErr bool + }{ + { + name: "Valid Proxy List", + args: args{ + proxyHosts: []string{"socks5://localhost:8080", "socks5://example.com:9090"}, + user: "user", + password: "password", + }, + want: []Proxy{ + { + Host: "localhost", + Port: 8080, + User: "user", + Password: "password", + }, + { + Host: "example.com", + Port: 9090, + User: "user", + Password: "password", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GenerateProxyList(tt.args.proxyHosts, tt.args.user, tt.args.password) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateProxyList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GenerateProxyList() = %v, want %v", got, tt.want) + } + }) + } +} From 492bb5f69c8dd505acf11d4a725e961fe44f6970 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Fri, 18 Jul 2025 22:06:33 +0100 Subject: [PATCH 2/3] feat: add proxy list to client --- client.go | 40 +++++++++++++++++++++++++++++++++------- client_test.go | 39 ++++++++++++++++++++++++++++++++++++++- example/main.go | 2 +- proxy.go | 10 ++++++++-- proxy_test.go | 9 +++++++-- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 1008591..db3c7e3 100644 --- a/client.go +++ b/client.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "net/http" + "net/url" + "sync" "time" "github.com/imroc/req/v3" @@ -13,8 +15,11 @@ import ( ) type Client struct { - client *req.Client - apiKey string + client *req.Client + apiKey string + proxies []Proxy + nextProxy int + mutex *sync.Mutex } func (c *Client) Client() *http.Client { @@ -106,6 +111,14 @@ func (c *Client) FetchTicketListings( return nil, fmt.Errorf("invalid input: %w", err) } + if len(c.proxies) > 0 { + proxyUrl, err := c.nextProxyUrl() + if err != nil { + return nil, fmt.Errorf("failed to get next proxy URL: %w", err) + } + c.client.SetProxyURL(proxyUrl.String()) + } + // Iterate through feeds until have equal to or more ticket listings than desired ticketListings := make(TicketListings, 0, input.MaxNumber) earliestTicketTime := input.CreatedBefore @@ -169,16 +182,29 @@ func (c *Client) FetchTicketListingsByFeedUrl( } // NewClient creates a new Twickets client -func NewClient(apiKey string) (*Client, error) { +func NewClient(apiKey string, proxies []Proxy) (*Client, error) { if apiKey == "" { return nil, errors.New("api key must be set") } client := req.C().ImpersonateChrome() - return &Client{ - client: client, - apiKey: apiKey, - }, nil + + c := &Client{ + client: client, + apiKey: apiKey, + proxies: proxies, + mutex: &sync.Mutex{}, + } + + return c, nil +} + +func (c *Client) nextProxyUrl() (*url.URL, error) { + c.mutex.Lock() + defer c.mutex.Unlock() + proxy := c.proxies[c.nextProxy] + c.nextProxy = (c.nextProxy + 1) % len(c.proxies) + return proxy.URL() } func sliceToMaxNumTicketListings(listings TicketListings, maxNumTicketListings int) TicketListings { diff --git a/client_test.go b/client_test.go index e3c500e..879b225 100644 --- a/client_test.go +++ b/client_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strconv" "testing" "github.com/ahobsonsayers/twigots" @@ -14,6 +15,42 @@ import ( "github.com/stretchr/testify/require" ) +func TestClientWithProxy(t *testing.T) { + testutils.SkipIfCI(t) + + projectDirectory := projectDirectory(t) + _ = godotenv.Load(filepath.Join(projectDirectory, ".env")) + + twicketsAPIKey := os.Getenv("TWICKETS_API_KEY") + require.NotEmpty(t, twicketsAPIKey, "TWICKETS_API_KEY is not set") + proxyUser := os.Getenv("PROXY_USER") + require.NotEmpty(t, proxyUser, "PROXY_USER is not set") + proxyPassword := os.Getenv("PROXY_PASSWORD") + require.NotEmpty(t, proxyPassword, "PROXY_PASSWORD is not set") + proxyHost := os.Getenv("PROXY_HOST") + require.NotEmpty(t, proxyHost, "PROXY_HOST is not set") + proxyPort := os.Getenv("PROXY_PORT") + require.NotEmpty(t, proxyPort, "PROXY_PORT is not set") + proxyPortInt, err := strconv.Atoi(proxyPort) + require.NoError(t, err, "failed to convert PROXY_PORT to int") + proxy, err := twigots.NewProxy(proxyHost, proxyPortInt, proxyUser, proxyPassword) + require.NoError(t, err) + require.NoError(t, err) + + twicketsClient, err := twigots.NewClient(twicketsAPIKey, []twigots.Proxy{*proxy}) + require.NoError(t, err) + + listings, err := twicketsClient.FetchTicketListings( + context.Background(), + twigots.FetchTicketListingsInput{ + Country: twigots.CountryUnitedKingdom, + MaxNumber: 10, + }, + ) + require.NoError(t, err) + spew.Dump(listings) +} + // TODO: Use httptest client func TestGetLatestTicketListings(t *testing.T) { @@ -25,7 +62,7 @@ func TestGetLatestTicketListings(t *testing.T) { twicketsAPIKey := os.Getenv("TWICKETS_API_KEY") require.NotEmpty(t, twicketsAPIKey, "TWICKETS_API_KEY is not set") - twicketsClient, err := twigots.NewClient(twicketsAPIKey) + twicketsClient, err := twigots.NewClient(twicketsAPIKey, nil) require.NoError(t, err) listings, err := twicketsClient.FetchTicketListings( diff --git a/example/main.go b/example/main.go index c290d26..1f381f3 100644 --- a/example/main.go +++ b/example/main.go @@ -14,7 +14,7 @@ func main() { apiKey := "my_api_key" // Create twickets client (using api key) - client, err := twigots.NewClient(apiKey) + client, err := twigots.NewClient(apiKey, nil) if err != nil { log.Fatal(err) } diff --git a/proxy.go b/proxy.go index 7092aa1..275ad1f 100644 --- a/proxy.go +++ b/proxy.go @@ -17,13 +17,19 @@ type Proxy struct { } // NewProxy creates a new Proxy instance with the given host, port, user, and password. -func NewProxy(host string, port int, user, password string) *Proxy { +func NewProxy(host string, port int, user, password string) (*Proxy, error) { + if host == "" { + return nil, errors.New("host is required") + } + if port <= 0 { + return nil, errors.New("port must be positive") + } return &Proxy{ Host: host, Port: port, User: user, Password: password, - } + }, nil } // GenerateProxyList creates a list of Proxy instances from a list of proxy URLs. diff --git a/proxy_test.go b/proxy_test.go index ca0592a..d285bf8 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -49,12 +49,17 @@ func TestNewProxy(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NewProxy( + got, err := NewProxy( tt.args.host, tt.args.port, tt.args.user, tt.args.password, - ); !reflect.DeepEqual(got, tt.want) { + ) + if err != nil { + t.Errorf("NewProxy() error = %v", err) + return + } + if !reflect.DeepEqual(got, tt.want) { t.Errorf("NewProxy() = %v, want %v", got, tt.want) } }) From 6defb19c92c8de6e2baac58951ae62e3fc15281b Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Sat, 19 Jul 2025 09:19:21 +0100 Subject: [PATCH 3/3] feat: add new client with proxy method to avoid changing existing interface --- client.go | 45 ++++++++++++++++++++++++++++++--------------- client_test.go | 4 ++-- example/main.go | 2 +- proxy.go | 3 +++ proxy_test.go | 2 +- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/client.go b/client.go index db3c7e3..4383b59 100644 --- a/client.go +++ b/client.go @@ -17,7 +17,7 @@ import ( type Client struct { client *req.Client apiKey string - proxies []Proxy + proxies *[]Proxy nextProxy int mutex *sync.Mutex } @@ -106,20 +106,14 @@ func (c *Client) FetchTicketListings( input FetchTicketListingsInput, ) (TicketListings, error) { input.applyDefaults() - err := input.Validate() - if err != nil { + if err := input.Validate(); err != nil { return nil, fmt.Errorf("invalid input: %w", err) } - if len(c.proxies) > 0 { - proxyUrl, err := c.nextProxyUrl() - if err != nil { - return nil, fmt.Errorf("failed to get next proxy URL: %w", err) - } - c.client.SetProxyURL(proxyUrl.String()) + if err := c.setupProxy(); err != nil { + return nil, err } - // Iterate through feeds until have equal to or more ticket listings than desired ticketListings := make(TicketListings, 0, input.MaxNumber) earliestTicketTime := input.CreatedBefore for (input.MaxNumber < 0 || len(ticketListings) < input.MaxNumber) && @@ -148,13 +142,24 @@ func (c *Client) FetchTicketListings( earliestTicketTime = feedTicketListings[len(feedTicketListings)-1].CreatedAt.Time } - // Only return ticket listings requested ticketListings = sliceToMaxNumTicketListings(ticketListings, input.MaxNumber) ticketListings = filterToCreatedAfter(ticketListings, input.CreatedAfter) return ticketListings, nil } +// setupProxy sets up the proxy for the client if proxies are configured. +func (c *Client) setupProxy() error { + if c.proxies != nil && len(*c.proxies) > 0 { + proxyUrl, err := c.nextProxyUrl() + if err != nil { + return fmt.Errorf("failed to get next proxy URL: %w", err) + } + c.client.SetProxyURL(proxyUrl.String()) + } + return nil +} + // FetchTicketListings gets ticket listings using the specified feel url. func (c *Client) FetchTicketListingsByFeedUrl( ctx context.Context, @@ -182,7 +187,7 @@ func (c *Client) FetchTicketListingsByFeedUrl( } // NewClient creates a new Twickets client -func NewClient(apiKey string, proxies []Proxy) (*Client, error) { +func NewClient(apiKey string) (*Client, error) { if apiKey == "" { return nil, errors.New("api key must be set") } @@ -192,18 +197,28 @@ func NewClient(apiKey string, proxies []Proxy) (*Client, error) { c := &Client{ client: client, apiKey: apiKey, - proxies: proxies, + proxies: nil, mutex: &sync.Mutex{}, } return c, nil } +// NewClientWithProxies creates a new Twickets client with a list of proxies. +func NewClientWithProxies(apiKey string, proxies *[]Proxy) (*Client, error) { + c, err := NewClient(apiKey) + if err != nil { + return nil, err + } + c.proxies = proxies + return c, nil +} + func (c *Client) nextProxyUrl() (*url.URL, error) { c.mutex.Lock() defer c.mutex.Unlock() - proxy := c.proxies[c.nextProxy] - c.nextProxy = (c.nextProxy + 1) % len(c.proxies) + proxy := (*c.proxies)[c.nextProxy] + c.nextProxy = (c.nextProxy + 1) % len(*c.proxies) return proxy.URL() } diff --git a/client_test.go b/client_test.go index 879b225..af142cc 100644 --- a/client_test.go +++ b/client_test.go @@ -37,7 +37,7 @@ func TestClientWithProxy(t *testing.T) { require.NoError(t, err) require.NoError(t, err) - twicketsClient, err := twigots.NewClient(twicketsAPIKey, []twigots.Proxy{*proxy}) + twicketsClient, err := twigots.NewClientWithProxies(twicketsAPIKey, &[]twigots.Proxy{*proxy}) require.NoError(t, err) listings, err := twicketsClient.FetchTicketListings( @@ -62,7 +62,7 @@ func TestGetLatestTicketListings(t *testing.T) { twicketsAPIKey := os.Getenv("TWICKETS_API_KEY") require.NotEmpty(t, twicketsAPIKey, "TWICKETS_API_KEY is not set") - twicketsClient, err := twigots.NewClient(twicketsAPIKey, nil) + twicketsClient, err := twigots.NewClient(twicketsAPIKey) require.NoError(t, err) listings, err := twicketsClient.FetchTicketListings( diff --git a/example/main.go b/example/main.go index 1f381f3..c290d26 100644 --- a/example/main.go +++ b/example/main.go @@ -14,7 +14,7 @@ func main() { apiKey := "my_api_key" // Create twickets client (using api key) - client, err := twigots.NewClient(apiKey, nil) + client, err := twigots.NewClient(apiKey) if err != nil { log.Fatal(err) } diff --git a/proxy.go b/proxy.go index 275ad1f..62c2140 100644 --- a/proxy.go +++ b/proxy.go @@ -36,6 +36,9 @@ func NewProxy(host string, port int, user, password string) (*Proxy, error) { func GenerateProxyList(proxyHosts []string, user, password string) ([]Proxy, error) { var proxyList []Proxy for _, p := range proxyHosts { + if !strings.Contains(p, "://") { + p = "socks5://" + p + } parsedUrl, err := url.Parse(p) if err != nil { return nil, fmt.Errorf("failed to parse proxy URL '%s': %w", p, err) diff --git a/proxy_test.go b/proxy_test.go index d285bf8..0440179 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -196,7 +196,7 @@ func TestGenerateProxyList(t *testing.T) { { name: "Valid Proxy List", args: args{ - proxyHosts: []string{"socks5://localhost:8080", "socks5://example.com:9090"}, + proxyHosts: []string{"localhost:8080", "example.com:9090"}, user: "user", password: "password", },