Feat retry ratelimit

config/config.go

@@ -54,6 +54,9 @@ type Config struct {
 	ReadBufferSize int           `env:"LONGBRIDGE_READ_BUFFER_SIZE,LONGPORT_READ_BUFFER_SIZE" yaml:"LONGBRIDGE_READ_BUFFER_SIZE,LONGPORT_READ_BUFFER_SIZE" toml:"LONGBRIDGE_READ_BUFFER_SIZE,LONGPORT_READ_BUFFER_SIZE"`
 	MinGzipSize    int           `env:"LONGBRIDGE_MIN_GZIP_SIZE,LONGPORT_MIN_GZIP_SIZE" yaml:"LONGBRIDGE_MIN_GZIP_SIZE,LONGPORT_MIN_GZIP_SIZE" toml:"LONGBRIDGE_MIN_GZIP_SIZE,LONGPORT_MIN_GZIP_SIZE"`
 	Region         Region        `env:"LONGPORT_REGION" yaml:"LONGPORT_REGION" toml:"LONGPORT_REGION"`
+
+	// DisalbeHTTPRetry user to disable http client retry request when rate limited by server
+	DisalbeHTTPRetry bool `env:"LONGPORT_HTTP_DISABLE_RETRY" yaml:"LONGPORT_HTTP_DISABLE_RETRY" toml:"LONGPORT_HTTP_DISABLE_RETRY"`
 }
 
 func (c *Config) SetLogger(l log.Logger) {
@@ -108,15 +111,15 @@ func New(opts ...Option) (configData *Config, err error) {
 
 func (c *Config) check() (err error) {
 	if c.AccessToken == "" {
-		err = errors.New("Don't has accessToken. Please set access token on LONGBRIDGE_ACCESS_TOKEN env")
+		err = errors.New("Don't has accessToken. Please set access token on LONGPORT_ACCESS_TOKEN env")
 		return
 	}
 	if c.AppKey == "" {
-		err = errors.New("Don't has appKey. Please set app key on LONGBRIDGE_APP_KEY env")
+		err = errors.New("Don't has appKey. Please set app key on LONGPORT_APP_KEY env")
 		return
 	}
 	if c.AppSecret == "" {
-		err = errors.New("Don't has appSecret. Please set app secret on LONGBRIDGE_APP_SECRET env")
+		err = errors.New("Don't has appSecret. Please set app secret on LONGPORT_APP_SECRET env")
 		return
 	}
 	return

examples/get_quote/main.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/longportapp/openapi-go/config"
 	"github.com/longportapp/openapi-go/quote"
@@ -20,6 +23,7 @@ func main() {
 		log.Fatal(err)
 		return
 	}
+	// close connection
 	defer quoteContext.Close()
 	ctx := context.Background()
 	// Get basic information of securities
@@ -28,7 +32,7 @@ func main() {
 		log.Fatal(err)
 		return
 	}
-	fmt.Printf("quotes: %v\n", quotes)
+	fmt.Printf("quotes: %+v\n", quotes[0])
 
 	warrants, err := quoteContext.WarrantList(ctx, "700.HK", quote.WarrantFilter{
 		SortBy:     quote.WarrantVolume,
@@ -41,5 +45,8 @@ func main() {
 		return
 	}
 
-	fmt.Printf("warrants: %v\n", warrants)
+	fmt.Printf("warrants: %+v\n", warrants[0])
+	quitChannel := make(chan os.Signal, 1)
+	signal.Notify(quitChannel, syscall.SIGINT, syscall.SIGTERM)
+	<-quitChannel
 }

examples/go.mod

@@ -3,7 +3,7 @@ module github.com/longportapp/openapi-go/examples
 go 1.17
 
 require (
-	github.com/longportapp/openapi-go v0.9.0
+	github.com/longportapp/openapi-go v0.12.0
 	github.com/shopspring/decimal v1.3.1
 )
 
@@ -16,7 +16,7 @@ require (
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/jinzhu/copier v0.3.5 // indirect
 	github.com/joho/godotenv v1.4.0 // indirect
-	github.com/longportapp/openapi-protobufs/gen/go v0.2.1 // indirect
+	github.com/longportapp/openapi-protobufs/gen/go v0.3.0 // indirect
 	github.com/longportapp/openapi-protocol/go v0.3.0 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/pkg/errors v0.9.1 // indirect

examples/go.sum

@@ -32,8 +32,11 @@ github.com/longbridgeapp/assert v0.1.0 h1:KkQlHUJSpuUFkUDjwBJgghFl31+wwSDHTq/WRr
 github.com/longbridgeapp/assert v0.1.0/go.mod h1:ew3umReliXtk1bBG4weVURxdvR0tsN+rCEfjnA4YfxI=
 github.com/longportapp/openapi-go v0.9.0 h1:ourTGssgbRNflW6SnMW/ewC3jJaa/vXFHVwFADULq00=
 github.com/longportapp/openapi-go v0.9.0/go.mod h1:wpoQEr+jgZ1+guzfh0BgYd3dT9n0eDByCc5gkfoVrkI=
+github.com/longportapp/openapi-go v0.12.0 h1:63cfkOiMMxhtcWSYRrqeS6Ia0z8UvlPFZM2odcPEf5A=
+github.com/longportapp/openapi-go v0.12.0/go.mod h1:vXazHVeSBuZAfnwDrejdTqgCD8TVgw2JYnScQ7f2IWU=
 github.com/longportapp/openapi-protobufs/gen/go v0.2.1 h1:AaubbUBGkawGYR4+XMorOIHr9Drte2CZBwjEKp6C1mU=
 github.com/longportapp/openapi-protobufs/gen/go v0.2.1/go.mod h1:/chiEwEW4CnOVgKTaCf8rQUwes00Ku8q1CvRpOueWfo=
+github.com/longportapp/openapi-protobufs/gen/go v0.3.0/go.mod h1:/chiEwEW4CnOVgKTaCf8rQUwes00Ku8q1CvRpOueWfo=
 github.com/longportapp/openapi-protocol/go v0.3.0 h1:Zv8YEkmkmbdZvbExunR5tHI8/DvjmidNK4vLy5ZHvUY=
 github.com/longportapp/openapi-protocol/go v0.3.0/go.mod h1:bO8FSq+4Iyg1UPZ5zoBS8V5xgSVXl0gA+Iw5nhWGpdo=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=

examples/submit_order/main.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/longportapp/openapi-go/config"
 	"github.com/longportapp/openapi-go/trade"
@@ -27,6 +30,7 @@ func main() {
 	// subscribe order status
 	tradeContext.OnTrade(func(ev *trade.PushEvent) {
 		// handle order changing event
+		log.Printf("order event: %+v\n", ev)
 	})
 
 	// submit order
@@ -44,4 +48,7 @@ func main() {
 		return
 	}
 	fmt.Printf("orderId: %v\n", orderId)
+	quitChannel := make(chan os.Signal, 1)
+	signal.Notify(quitChannel, syscall.SIGINT, syscall.SIGTERM)
+	<-quitChannel
 }

http/client.go

@@ -6,16 +6,23 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"net/http"
 	nhttp "net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/google/go-querystring/query"
 
 	"github.com/longportapp/openapi-go/config"
 	"github.com/longportapp/openapi-go/log"
 )
 
+const (
+	headerRetryAfter = "retry-after"
+	headerTraceID    = "x-trace-id"
+)
+
 type apiResponse struct {
 	Code    int
 	Message string
@@ -27,6 +34,16 @@ type otpResponse struct {
 	Otp string
 }
 
+var _ error = (*retryError)(nil)
+
+type retryError struct{}
+
+func (re *retryError) Error() string {
+	return "retry"
+}
+
+var errNeedRetry = &retryError{}
+
 // Client is a http client to access Longbridge REST OpenAPI
 type Client struct {
 	opts       *Options
@@ -145,22 +162,65 @@ func (c *Client) Call(ctx context.Context, method, path string, queryParams inte
 		}
 		req.URL.RawQuery = vals.Encode()
 	}
-	req.Header.Add("x-api-key", c.opts.AppKey)
-	req.Header.Add("authorization", c.opts.AccessToken)
+	req.Header.Set("x-api-key", c.opts.AppKey)
+	req.Header.Set("authorization", c.opts.AccessToken)
 	if len(bb) != 0 {
-		req.Header.Add("content-type", "application/json; charset=utf-8")
+		req.Header.Set("content-type", "application/json; charset=utf-8")
 	}
-	signature(req, c.opts.AppSecret, bb)
 
-	log.Debugf("http call method:%v url:%v body:%v", req.Method, req.URL, string(bb))
-	req.Close = true
-	httpResp, err = c.httpClient.Do(req)
-	if err != nil {
-		return err
+	log.Debugf("http call method:%v url:%v body:%s", req.Method, req.URL, bb)
+	var retryCount int
+	call := func(disabeRetry bool) error {
+		if retryCount > 0 {
+			log.Debugf("retry calling %s %s, count: %d", req.Method, req.URL.Path, retryCount)
+			// reset x-timestamp in signature func
+			req.Header.Set(headerTimestamp, "")
+		}
+
+		signature(req, c.opts.AppSecret, bb)
+
+		httpResp, err = c.httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+
+		// if disabled retry just return
+		if disabeRetry {
+			return nil
+		}
+
+		wait, ok := parseRatelimit(httpResp)
+
+		if !ok {
+			// if not rate limited just return
+			return nil
+		}
+
+		// need retry so close body
+		httpResp.Body.Close()
+		time.Sleep(wait)
+		return errNeedRetry
+	}
+
+	for {
+		err = call(c.opts.DisableRetry)
+
+		if err == nil {
+			break
+		}
+
+		if errors.Is(err, errNeedRetry) {
+			retryCount = retryCount + 1
+			// retry
+			continue
+
+		}
+		// handle error
+		break
 	}
+
 	log.Debugf("http call response headers:%v", httpResp.Header)
 	defer httpResp.Body.Close()
-
 	if rb, err = io.ReadAll(httpResp.Body); err != nil {
 		return err
 	}
@@ -198,13 +258,37 @@ func isJSON(ct string) bool {
 	return strings.Contains(ct, "application/json")
 }
 
+func parseRatelimit(res *nhttp.Response) (time.Duration, bool) {
+	if res.StatusCode != http.StatusTooManyRequests {
+		return 0, false
+	}
+
+	waitStr := res.Header.Get(headerRetryAfter)
+	if waitStr == "" {
+		return 0, false
+	}
+
+	d, err := time.ParseDuration(waitStr)
+	if err != nil {
+		return 0, false
+	}
+
+	log.Warnf("request %s %s (%s) has been rate limited, will retry after %v...", res.Request.Method, res.Request.URL.Path, res.Header.Get(headerTraceID), d)
+
+	return d, true
+}
+
 func jsonUnmarshal(r io.Reader, v interface{}) error {
 	d := json.NewDecoder(r)
 	d.UseNumber()
 	return d.Decode(v)
 }
 
 // New create http client to call Longbridge REST OpenAPI
+//
+// Example:
+//	cli, err := New(http.WithAppKey("appkey"), http.WithAppSecret("appSecret"), http.WithAccessToken("token"))
+
 func New(opt ...Option) (*Client, error) {
 	opts := newOptions(opt...)
 	if opts.URL == "" {
@@ -226,12 +310,20 @@ func New(opt ...Option) (*Client, error) {
 
 // NewFromCfg init longbridge http client from *config.Config
 func NewFromCfg(c *config.Config) (*Client, error) {
-	return New(
+	cli, err := New(
 		WithAccessToken(c.AccessToken),
 		WithAppKey(c.AppKey),
 		WithAppSecret(c.AppSecret),
 		WithTimeout(c.HTTPTimeout),
 		WithClient(c.Client),
 		WithURL(c.HttpURL),
 	)
+	if err != nil {
+		return cli, err
+	}
+
+	if c.DisalbeHTTPRetry {
+		cli.opts.DisableRetry = true
+	}
+	return cli, nil
 }

http/options.go

@@ -13,12 +13,13 @@ const DefaultTimeout = 15 * time.Second
 
 // Options for http client
 type Options struct {
-	URL         string
-	AppKey      string
-	AppSecret   string
-	AccessToken string
-	Timeout     time.Duration
-	Client      *http.Client
+	URL          string
+	AppKey       string
+	AppSecret    string
+	AccessToken  string
+	Timeout      time.Duration
+	Client       *http.Client
+	DisableRetry bool
 }
 
 // Option for http client
@@ -78,6 +79,14 @@ func WithTimeout(timeout time.Duration) Option {
 	}
 }
 
+// DisableRetry can disable retry calling when last request is rate limited by server.
+// Client default retry call api after rate limited by server
+func DisableRetry() Option {
+	return func(opts *Options) {
+		opts.DisableRetry = true
+	}
+}
+
 func newOptions(opt ...Option) *Options {
 	opts := Options{
 		Timeout: DefaultTimeout,

http/signature.go

@@ -18,10 +18,10 @@ var sign = &signer.Signer{}
 
 func signature(req *nhttp.Request, secret string, body []byte) error {
 	if v := req.Header.Get(headerTimestamp); v == "" {
-		req.Header.Add(headerTimestamp, strconv.FormatInt(time.Now().UnixMilli(), 10))
+		req.Header.Set(headerTimestamp, strconv.FormatInt(time.Now().UnixMilli(), 10))
 	}
 
-	req.Header.Add("x-api-signature", "HMAC-SHA256 SignedHeaders=authorization;x-api-key;x-timestamp")
+	req.Header.Set("x-api-signature", "HMAC-SHA256 SignedHeaders=authorization;x-api-key;x-timestamp")
 	signstr, _, _, err := sign.Sign(context.Background(), util.UnsafeStringToBytes(secret), req, body)
 	if err != nil {
 		return err