From 5dbb5c5fb7a2fcdc323cb743e2b54d20ed5454c8 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 14:54:54 +0900 Subject: [PATCH 01/26] extend api for proxy --- coordinator/internal/controller/api/auth.go | 41 +++++++++++++++------ coordinator/internal/logic/auth/login.go | 24 ++++++++---- coordinator/internal/types/prover.go | 4 ++ 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index b38f0d827a..cdbb8225d8 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -26,21 +26,43 @@ func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) * } } -// Login the api controller for login +// Login the api controller for login, used as the Authenticator in JWT +// It can work in two mode: full process for normal login, or if login request +// is posted from proxy, run a simpler process to login a client func (a *AuthController) Login(c *gin.Context) (interface{}, error) { + + // check if the login is post by proxy + var viaProxy bool + if proverType, proverTypeExist := c.Get(types.ProverProviderTypeKey); proverTypeExist { + proverType := uint8(proverType.(float64)) + viaProxy = proverType == types.ProverProviderTypeProxy + } + var login types.LoginParameter if err := c.ShouldBind(&login); err != nil { return "", fmt.Errorf("missing the public_key, err:%w", err) } - // check login parameter's token is equal to bearer token, the Authorization must be existed - // if not exist, the jwt token will intercept it - brearToken := c.GetHeader("Authorization") - if brearToken != "Bearer "+login.Message.Challenge { - return "", errors.New("check challenge failure for the not equal challenge string") + // if not, process with normal login + if !viaProxy { + // check login parameter's token is equal to bearer token, the Authorization must be existed + // if not exist, the jwt token will intercept it + brearToken := c.GetHeader("Authorization") + if brearToken != "Bearer "+login.Message.Challenge { + return "", errors.New("check challenge failure for the not equal challenge string") + } + + if err := a.loginLogic.VerifyMsg(&login); err != nil { + return "", err + } + + // check the challenge is used, if used, return failure + if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil { + return "", fmt.Errorf("login insert challenge string failure:%w", err) + } } - if err := a.loginLogic.Check(&login); err != nil { + if err := a.loginLogic.CompatiblityCheck(&login); err != nil { return "", fmt.Errorf("check the login parameter failure: %w", err) } @@ -49,11 +71,6 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return "", fmt.Errorf("prover hard fork name failure:%w", err) } - // check the challenge is used, if used, return failure - if err := a.loginLogic.InsertChallengeString(c, login.Message.Challenge); err != nil { - return "", fmt.Errorf("login insert challenge string failure:%w", err) - } - returnData := types.LoginParameterWithHardForkName{ HardForkName: hardForkNames, LoginParameter: login, diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index f5c01a2dfe..0d9222dff6 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -50,13 +50,19 @@ func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) e return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) } -func (l *LoginLogic) Check(login *types.LoginParameter) error { +// Verify the completeness of login message +func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { verify, err := login.Verify() if err != nil || !verify { log.Error("auth message verify failure", "prover_name", login.Message.ProverName, "prover_version", login.Message.ProverVersion, "message", login.Message) return errors.New("auth message verify failure") } + return nil +} + +// Check if the login client is compatible with the setting in coordinator +func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.ProverManager.Verifier.MinProverVersion) { return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.ProverManager.Verifier.MinProverVersion, login.Message.ProverVersion) @@ -80,14 +86,16 @@ func (l *LoginLogic) Check(login *types.LoginParameter) error { } } - if login.Message.ProverProviderType != types.ProverProviderTypeInternal && login.Message.ProverProviderType != types.ProverProviderTypeExternal { + switch login.Message.ProverProviderType { + case types.ProverProviderTypeInternal: + case types.ProverProviderTypeExternal: + case types.ProverProviderTypeProxy: + case types.ProverProviderTypeUndefined: // for backward compatibility, set ProverProviderType as internal - if login.Message.ProverProviderType == types.ProverProviderTypeUndefined { - login.Message.ProverProviderType = types.ProverProviderTypeInternal - } else { - log.Error("invalid prover_provider_type", "value", login.Message.ProverProviderType, "prover name", login.Message.ProverName, "prover version", login.Message.ProverVersion) - return errors.New("invalid prover provider type.") - } + login.Message.ProverProviderType = types.ProverProviderTypeInternal + default: + log.Error("invalid prover_provider_type", "value", login.Message.ProverProviderType, "prover name", login.Message.ProverName, "prover version", login.Message.ProverVersion) + return errors.New("invalid prover provider type.") } return nil diff --git a/coordinator/internal/types/prover.go b/coordinator/internal/types/prover.go index 048fac00a2..4254c673d5 100644 --- a/coordinator/internal/types/prover.go +++ b/coordinator/internal/types/prover.go @@ -64,6 +64,8 @@ func (r ProverProviderType) String() string { return "prover provider type internal" case ProverProviderTypeExternal: return "prover provider type external" + case ProverProviderTypeProxy: + return "prover provider type proxy" default: return fmt.Sprintf("prover provider type: %d", r) } @@ -76,4 +78,6 @@ const ( ProverProviderTypeInternal // ProverProviderTypeExternal is an external prover provider type ProverProviderTypeExternal + // ProverProviderTypeProxy is an proxy prover provider type + ProverProviderTypeProxy = 3 ) From 1f2b85767179ee14d92e59b8debb30f6474fdd50 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 15:35:51 +0900 Subject: [PATCH 02/26] add proxy_login route --- coordinator/internal/route/route.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 9e9eef076e..2a0383aa8d 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -34,6 +34,7 @@ func v1(router *gin.RouterGroup, conf *config.Config) { // need jwt token api r.Use(loginMiddleware.MiddlewareFunc()) { + r.POST("/proxy_login", loginMiddleware.LoginHandler) r.POST("/get_task", api.GetTask.GetTasks) r.POST("/submit_proof", api.SubmitProof.SubmitProof) } From 9796d16f6c68d3811303cedafcf0d11816823591 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 20:32:11 +0900 Subject: [PATCH 03/26] WIP: update login logic and coordinator client --- coordinator/internal/controller/api/auth.go | 4 +- .../internal/controller/proxy/client.go | 214 ++++++++++++++++++ coordinator/internal/logic/auth/login.go | 26 +-- 3 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 coordinator/internal/controller/proxy/client.go diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index cdbb8225d8..1a3305d2b1 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -22,7 +22,7 @@ type AuthController struct { // NewAuthController returns an LoginController instance func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *AuthController { return &AuthController{ - loginLogic: auth.NewLoginLogic(db, cfg, vf), + loginLogic: auth.NewLoginLogic(db, cfg.ProverManager.Verifier, vf), } } @@ -52,7 +52,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return "", errors.New("check challenge failure for the not equal challenge string") } - if err := a.loginLogic.VerifyMsg(&login); err != nil { + if err := auth.VerifyMsg(&login); err != nil { return "", err } diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go new file mode 100644 index 0000000000..48d768ba80 --- /dev/null +++ b/coordinator/internal/controller/proxy/client.go @@ -0,0 +1,214 @@ +package proxy + +import ( + "bytes" + "crypto/ecdsa" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + + "scroll-tech/coordinator/internal/types" +) + +// Client wraps an http client with a preset host for coordinator API calls +type Client struct { + httpClient *http.Client + host string + loginToken string +} + +// NewClient creates a new Client with the specified host +func NewClient(host string) *Client { + return &Client{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + host: host, + } +} + +// NewClientWithHTTPClient creates a new Client with a custom http.Client +func NewClientWithHTTPClient(host string, httpClient *http.Client) *Client { + return &Client{ + httpClient: httpClient, + host: host, + } +} + +// FullLogin performs the complete login process: get challenge then login +func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { + // Step 1: Get challenge + url := fmt.Sprintf("%s/v1/challenge", c.host) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create challenge request: %w", err) + } + + challengeResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get challenge: %w", err) + } + defer challengeResp.Body.Close() + + if challengeResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("challenge request failed with status: %d", challengeResp.StatusCode) + } + + // Step 2: Parse challenge response + var loginSchema types.LoginSchema + if err := json.NewDecoder(challengeResp.Body).Decode(&loginSchema); err != nil { + return nil, fmt.Errorf("failed to parse challenge response: %w", err) + } + + // Step 3: Use the token from challenge as Bearer token for login + url = fmt.Sprintf("%s/v1/login", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal login parameter: %w", err) + } + + req, err = http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+loginSchema.Token) + + loginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform login request: %w", err) + } + + // Parse login response as LoginSchema and store the token + if loginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(loginResp.Body).Decode(&loginResult); err == nil { + c.loginToken = loginResult.Token + } + // Note: Body is consumed after decoding, caller should not read it again + return &loginResult, nil + } + + return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) +} + +// ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter +func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/proxy_login", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create proxy login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.loginToken) + + return c.httpClient.Do(req) +} + +// GetTask makes a POST request to /v1/get_task with GetTaskParameter +func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/get_task", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create get task request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter +func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/v1/submit_proof", c.host) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create submit proof request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes +func (c *Client) buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { + // Try appending bytes from 0x0 to 0x20 until we get a valid private key + for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { + // Append the byte to input + extendedBytes := append(inputBytes, appendByte) + + // Calculate 256-bit hash + hash := crypto.Keccak256(extendedBytes) + + // Try to create private key from hash + if k, err := crypto.ToECDSA(hash); err == nil { + return k, nil + } + } + + return nil, fmt.Errorf("failed to generate valid private key from input bytes") +} + +func (c *Client) generateLoginParameter(privateKeyBytes []byte, challenge string) (*types.LoginParameter, error) { + // Generate private key + privKey, err := c.buildPrivateKey(privateKeyBytes) + if err != nil { + return nil, err + } + + // Generate public key string + publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&privKey.PublicKey)) + + // Create login parameter with proxy settings + loginParam := &types.LoginParameter{ + Message: types.Message{ + Challenge: challenge, + ProverName: "proxy", + ProverVersion: "proxy", + ProverProviderType: types.ProverProviderTypeProxy, + ProverTypes: []types.ProverType{}, // Default empty + VKs: []string{}, // Default empty + }, + PublicKey: publicKeyHex, + } + + // Sign the message with the private key + if err := loginParam.SignWithKey(privKey); err != nil { + return nil, fmt.Errorf("failed to sign login parameter: %w", err) + } + + return loginParam, nil +} diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 0d9222dff6..35350ef1ea 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -19,7 +19,7 @@ import ( // LoginLogic the auth logic type LoginLogic struct { - cfg *config.Config + cfg *config.VerifierConfig challengeOrm *orm.Challenge openVmVks map[string]struct{} @@ -28,30 +28,25 @@ type LoginLogic struct { } // NewLoginLogic new a LoginLogic -func NewLoginLogic(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *LoginLogic { +func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { proverVersionHardForkMap := make(map[string][]string) var hardForks []string - for _, cfg := range cfg.ProverManager.Verifier.Verifiers { + for _, cfg := range vcfg.Verifiers { hardForks = append(hardForks, cfg.ForkName) } - proverVersionHardForkMap[cfg.ProverManager.Verifier.MinProverVersion] = hardForks + proverVersionHardForkMap[vcfg.MinProverVersion] = hardForks return &LoginLogic{ - cfg: cfg, + cfg: vcfg, openVmVks: vf.OpenVMVkMap, challengeOrm: orm.NewChallenge(db), proverVersionHardForkMap: proverVersionHardForkMap, } } -// InsertChallengeString insert and check the challenge string is existed -func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { - return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) -} - // Verify the completeness of login message -func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { +func VerifyMsg(login *types.LoginParameter) error { verify, err := login.Verify() if err != nil || !verify { log.Error("auth message verify failure", "prover_name", login.Message.ProverName, @@ -61,11 +56,16 @@ func (l *LoginLogic) VerifyMsg(login *types.LoginParameter) error { return nil } +// InsertChallengeString insert and check the challenge string is existed +func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { + return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) +} + // Check if the login client is compatible with the setting in coordinator func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { - if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.ProverManager.Verifier.MinProverVersion) { - return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.ProverManager.Verifier.MinProverVersion, login.Message.ProverVersion) + if !version.CheckScrollRepoVersion(login.Message.ProverVersion, l.cfg.MinProverVersion) { + return fmt.Errorf("incompatible prover version. please upgrade your prover, minimum allowed version: %s, actual version: %s", l.cfg.MinProverVersion, login.Message.ProverVersion) } vks := make(map[string]struct{}) From 412ad56a64677e9175b8a2d37523d35a9ec4723c Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 20:43:40 +0900 Subject: [PATCH 04/26] extend loginlogic --- coordinator/internal/logic/auth/login.go | 27 +++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 35350ef1ea..12830d6d4d 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -1,6 +1,7 @@ package auth import ( + "context" "errors" "fmt" "strings" @@ -20,15 +21,35 @@ import ( // LoginLogic the auth logic type LoginLogic struct { cfg *config.VerifierConfig - challengeOrm *orm.Challenge + deduplicator ChallengeDeduplicator openVmVks map[string]struct{} proverVersionHardForkMap map[string][]string } +type ChallengeDeduplicator interface { + InsertChallenge(ctx context.Context, challengeString string) error +} + +type SimpleDeduplicator struct { +} + +func (s *SimpleDeduplicator) InsertChallenge(ctx context.Context, challengeString string) error { + return nil +} + +// NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challege +func NewLoginLogicWithSimpleDEduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { + return newLoginLogic(&SimpleDeduplicator{}, vcfg, vf) +} + // NewLoginLogic new a LoginLogic func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { + return newLoginLogic(orm.NewChallenge(db), vcfg, vf) +} + +func newLoginLogic(deduplicator ChallengeDeduplicator, vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { proverVersionHardForkMap := make(map[string][]string) var hardForks []string @@ -40,7 +61,7 @@ func NewLoginLogic(db *gorm.DB, vcfg *config.VerifierConfig, vf *verifier.Verifi return &LoginLogic{ cfg: vcfg, openVmVks: vf.OpenVMVkMap, - challengeOrm: orm.NewChallenge(db), + deduplicator: deduplicator, proverVersionHardForkMap: proverVersionHardForkMap, } } @@ -58,7 +79,7 @@ func VerifyMsg(login *types.LoginParameter) error { // InsertChallengeString insert and check the challenge string is existed func (l *LoginLogic) InsertChallengeString(ctx *gin.Context, challenge string) error { - return l.challengeOrm.InsertChallenge(ctx.Copy(), challenge) + return l.deduplicator.InsertChallenge(ctx.Copy(), challenge) } // Check if the login client is compatible with the setting in coordinator From 3adb2e0a1b123c49179711cf8b89cd8041651750 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 21:18:13 +0900 Subject: [PATCH 05/26] WIP: controller --- coordinator/internal/config/proxy_config.go | 53 +++++++++++++++++++ .../internal/controller/proxy/client.go | 27 ++++------ .../internal/controller/proxy/controller.go | 41 ++++++++++++++ 3 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 coordinator/internal/config/proxy_config.go create mode 100644 coordinator/internal/controller/proxy/controller.go diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go new file mode 100644 index 0000000000..f94ef54c2e --- /dev/null +++ b/coordinator/internal/config/proxy_config.go @@ -0,0 +1,53 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + + "scroll-tech/common/utils" +) + +// Proxy loads proxy configuration items. +type ProxyManager struct { + // Zk verifier config help to confine the connected prover. + Verifier *VerifierConfig `json:"verifier"` +} + +// Coordinator configuration +type UpStream struct { + BaseUrl string `json:"base_url"` + RetryCount uint `json:"retry_count"` + RetryWaitTime uint `json:"retry_wait_time_sec"` + ConnectionTimeoutSec uint `json:"connection_timeout_sec"` +} + +// Config load configuration items. +type ProxyConfig struct { + ProxyManager *ProxyManager `json:"proxy_manager"` + ProxyName string `json:"proxy_name"` + Auth *Auth `json:"auth"` + Coordinators map[string]*UpStream `json:"coondiators"` +} + +// NewConfig returns a new instance of Config. +func NewProxyConfig(file string) (*ProxyConfig, error) { + buf, err := os.ReadFile(filepath.Clean(file)) + if err != nil { + return nil, err + } + + cfg := &ProxyConfig{} + err = json.Unmarshal(buf, cfg) + if err != nil { + return nil, err + } + + // Override config with environment variables + err = utils.OverrideConfigWithEnv(cfg, "SCROLL_COORDINATOR_PROXY") + if err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 48d768ba80..17c394db68 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -11,38 +11,31 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" + "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) // Client wraps an http client with a preset host for coordinator API calls type Client struct { httpClient *http.Client - host string + baseURL string loginToken string } // NewClient creates a new Client with the specified host -func NewClient(host string) *Client { +func NewClient(cfg *config.UpStream) *Client { return &Client{ httpClient: &http.Client{ - Timeout: 30 * time.Second, + Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, - host: host, - } -} - -// NewClientWithHTTPClient creates a new Client with a custom http.Client -func NewClientWithHTTPClient(host string, httpClient *http.Client) *Client { - return &Client{ - httpClient: httpClient, - host: host, + baseURL: cfg.BaseUrl, } } // FullLogin performs the complete login process: get challenge then login func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { // Step 1: Get challenge - url := fmt.Sprintf("%s/v1/challenge", c.host) + url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -66,7 +59,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { } // Step 3: Use the token from challenge as Bearer token for login - url = fmt.Sprintf("%s/v1/login", c.host) + url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -101,7 +94,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/proxy_login", c.host) + url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -121,7 +114,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) // GetTask makes a POST request to /v1/get_task with GetTaskParameter func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/get_task", c.host) + url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { @@ -143,7 +136,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/v1/submit_proof", c.host) + url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) if err != nil { diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go new file mode 100644 index 0000000000..02f6350d7d --- /dev/null +++ b/coordinator/internal/controller/proxy/controller.go @@ -0,0 +1,41 @@ +package proxy + +import ( + "github.com/scroll-tech/go-ethereum/log" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/verifier" +) + +var ( + // GetTask the prover task controller + GetTask *GetTaskController + // SubmitProof the submit proof controller + SubmitProof *SubmitProofController + // Auth the auth controller + Auth *AuthController +) + +// Clients manager a series of thread-safe clients for requesting upstream +// coordinators +type Clients map[string]*Client + +// InitController inits Controller with database +func InitController(cfg *config.ProxyConfig) { + vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) + if err != nil { + panic("proof receiver new verifier failure") + } + + log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) + + clients := make(map[string]*Client) + + for nm, cfg := range cfg.Coordinators { + clients[nm] = NewClient(cfg) + } + + Auth = NewAuthController(cfg, clients, vf) + // GetTask = NewGetTaskController(cfg, chainCfg, db, vf, reg) + // SubmitProof = NewSubmitProofController(cfg, chainCfg, db, vf, reg) +} From 5c6c225f7628f6ecbc5a9a88a07ede7d7642d8a8 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:14:22 +0900 Subject: [PATCH 06/26] WIP: config and client controller --- coordinator/internal/config/proxy_config.go | 19 ++++ .../internal/controller/proxy/client.go | 75 +++------------- .../controller/proxy/client_manager.go | 86 +++++++++++++++++++ .../internal/controller/proxy/controller.go | 15 +++- 4 files changed, 128 insertions(+), 67 deletions(-) create mode 100644 coordinator/internal/controller/proxy/client_manager.go diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index f94ef54c2e..5edb7fc783 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -12,6 +12,25 @@ import ( type ProxyManager struct { // Zk verifier config help to confine the connected prover. Verifier *VerifierConfig `json:"verifier"` + Client *ProxyClient `json:"proxy_cli"` + Auth *Auth `json:"auth"` +} + +func (m *ProxyManager) Normalize() { + if m.Client.Auth == nil { + m.Client.Auth = m.Auth + } + + if m.Client.ProxyVersion == "" { + m.Client.ProxyVersion = m.Verifier.MinProverVersion + } +} + +// Proxy client configuration for connect to upstream as a client +type ProxyClient struct { + ProxyName string `json:"proxy_name"` + ProxyVersion string `json:"proxy_version,omitempty"` + Auth *Auth `json:"auth,omitempty"` } // Coordinator configuration diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 17c394db68..9fe5ea8cb7 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -2,29 +2,28 @@ package proxy import ( "bytes" - "crypto/ecdsa" + "context" "encoding/json" "fmt" "net/http" "time" - "github.com/scroll-tech/go-ethereum/common" - "github.com/scroll-tech/go-ethereum/crypto" + "github.com/gin-gonic/gin" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) // Client wraps an http client with a preset host for coordinator API calls -type Client struct { +type upClient struct { httpClient *http.Client baseURL string loginToken string } // NewClient creates a new Client with the specified host -func NewClient(cfg *config.UpStream) *Client { - return &Client{ +func newUpClient(cfg *config.UpStream) *upClient { + return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, @@ -33,7 +32,7 @@ func NewClient(cfg *config.UpStream) *Client { } // FullLogin performs the complete login process: get challenge then login -func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -93,7 +92,7 @@ func (c *Client) Login(param types.LoginParameter) (*types.LoginSchema, error) { } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) { +func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -101,7 +100,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create proxy login request: %w", err) } @@ -113,7 +112,7 @@ func (c *Client) ProxyLogin(param types.LoginParameter) (*http.Response, error) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -121,7 +120,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create get task request: %w", err) } @@ -135,7 +134,7 @@ func (c *Client) GetTask(param types.GetTaskParameter, token string) (*http.Resp } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx *gin.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) @@ -143,7 +142,7 @@ func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*h return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) } - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create submit proof request: %w", err) } @@ -155,53 +154,3 @@ func (c *Client) SubmitProof(param types.SubmitProofParameter, token string) (*h return c.httpClient.Do(req) } - -// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes -func (c *Client) buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { - // Try appending bytes from 0x0 to 0x20 until we get a valid private key - for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { - // Append the byte to input - extendedBytes := append(inputBytes, appendByte) - - // Calculate 256-bit hash - hash := crypto.Keccak256(extendedBytes) - - // Try to create private key from hash - if k, err := crypto.ToECDSA(hash); err == nil { - return k, nil - } - } - - return nil, fmt.Errorf("failed to generate valid private key from input bytes") -} - -func (c *Client) generateLoginParameter(privateKeyBytes []byte, challenge string) (*types.LoginParameter, error) { - // Generate private key - privKey, err := c.buildPrivateKey(privateKeyBytes) - if err != nil { - return nil, err - } - - // Generate public key string - publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&privKey.PublicKey)) - - // Create login parameter with proxy settings - loginParam := &types.LoginParameter{ - Message: types.Message{ - Challenge: challenge, - ProverName: "proxy", - ProverVersion: "proxy", - ProverProviderType: types.ProverProviderTypeProxy, - ProverTypes: []types.ProverType{}, // Default empty - VKs: []string{}, // Default empty - }, - PublicKey: publicKeyHex, - } - - // Sign the message with the private key - if err := loginParam.SignWithKey(privKey); err != nil { - return nil, fmt.Errorf("failed to sign login parameter: %w", err) - } - - return loginParam, nil -} diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go new file mode 100644 index 0000000000..302292fb16 --- /dev/null +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -0,0 +1,86 @@ +package proxy + +import ( + "context" + "crypto/ecdsa" + "fmt" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/types" +) + +type Client interface { + Client(context.Context) *upClient +} + +type ClientManager struct { + cliCfg *config.ProxyClient + cfg *config.UpStream + privKey *ecdsa.PrivateKey +} + +// transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes +func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { + // Try appending bytes from 0x0 to 0x20 until we get a valid private key + for appendByte := byte(0x0); appendByte <= 0x20; appendByte++ { + // Append the byte to input + extendedBytes := append(inputBytes, appendByte) + + // Calculate 256-bit hash + hash := crypto.Keccak256(extendedBytes) + + // Try to create private key from hash + if k, err := crypto.ToECDSA(hash); err == nil { + return k, nil + } + } + + return nil, fmt.Errorf("failed to generate valid private key from input bytes") +} + +func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { + + privKey, err := buildPrivateKey([]byte(cliCfg.Auth.Secret)) + if err != nil { + return nil, err + } + + return &ClientManager{ + privKey: privKey, + cfg: cfg, + cliCfg: cliCfg, + }, nil +} + +func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { + return newUpClient(cliMgr.cfg) +} + +func (cliMgr *ClientManager) generateLoginParameter(privKey []byte, challenge string) (*types.LoginParameter, error) { + + // Generate public key string + publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) + + // Create login parameter with proxy settings + loginParam := &types.LoginParameter{ + Message: types.Message{ + Challenge: challenge, + ProverName: cliMgr.cliCfg.ProxyName, + ProverVersion: cliMgr.cliCfg.ProxyVersion, + ProverProviderType: types.ProverProviderTypeProxy, + ProverTypes: []types.ProverType{}, // Default empty + VKs: []string{}, // Default empty + }, + PublicKey: publicKeyHex, + } + + // Sign the message with the private key + if err := loginParam.SignWithKey(cliMgr.privKey); err != nil { + return nil, fmt.Errorf("failed to sign login parameter: %w", err) + } + + return loginParam, nil +} diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 02f6350d7d..297f8c68a4 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -18,10 +18,13 @@ var ( // Clients manager a series of thread-safe clients for requesting upstream // coordinators -type Clients map[string]*Client +type Clients map[string]Client // InitController inits Controller with database func InitController(cfg *config.ProxyConfig) { + // normalize cfg + cfg.ProxyManager.Normalize() + vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) if err != nil { panic("proof receiver new verifier failure") @@ -29,10 +32,14 @@ func InitController(cfg *config.ProxyConfig) { log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) - clients := make(map[string]*Client) + clients := make(map[string]Client) - for nm, cfg := range cfg.Coordinators { - clients[nm] = NewClient(cfg) + for nm, upCfg := range cfg.Coordinators { + cli, err := NewClientManager(cfg.ProxyManager.Client, upCfg) + if err != nil { + panic("create new client fail") + } + clients[nm] = cli } Auth = NewAuthController(cfg, clients, vf) From 76ecdf064a79bac7a645bd9c9afcde50bcf21d07 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:14:32 +0900 Subject: [PATCH 07/26] add proxy config sample --- coordinator/conf/config_proxy.json | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 coordinator/conf/config_proxy.json diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json new file mode 100644 index 0000000000..39c3ca0a5f --- /dev/null +++ b/coordinator/conf/config_proxy.json @@ -0,0 +1,33 @@ +{ + "proxy_manager": { + "proxy_cli": { + "proxy_name": "proxy_name" + }, + "auth": { + "secret": "proxy secret key", + "challenge_expire_duration_sec": 3600, + "login_expire_duration_sec": 3600 + }, + "verifier": { + "min_prover_version": "v4.4.45", + "verifiers": [ + { + "assets_path": "assets", + "fork_name": "euclidV2" + }, + { + "assets_path": "assets", + "fork_name": "feynman" + } + ] + } + }, + "coordinators": { + "sepolia": { + "base_url": "http://localhost:8555", + "retry_count": 10, + "retry_wait_time_sec": 10, + "connection_timeout_sec": 30 + } + } +} From 0d238d77a6ddb1c0f66030f9154ffb45c9092d80 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:32:38 +0900 Subject: [PATCH 08/26] WIP: the structure of client manager --- coordinator/internal/controller/proxy/client.go | 16 ++++++++++++++-- .../internal/controller/proxy/client_manager.go | 8 ++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 9fe5ea8cb7..8850274f1c 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -14,25 +14,32 @@ import ( "scroll-tech/coordinator/internal/types" ) +type ClientHelper interface { + GenLoginParam(string) (*types.LoginParameter, error) + OnError(isUnauth bool) +} + // Client wraps an http client with a preset host for coordinator API calls type upClient struct { httpClient *http.Client baseURL string loginToken string + helper ClientHelper } // NewClient creates a new Client with the specified host -func newUpClient(cfg *config.UpStream) *upClient { +func newUpClient(cfg *config.UpStream, helper ClientHelper) *upClient { return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, baseURL: cfg.BaseUrl, + helper: helper, } } // FullLogin performs the complete login process: get challenge then login -func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -60,6 +67,11 @@ func (c *upClient) Login(ctx context.Context, param types.LoginParameter) (*type // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) + param, err := c.helper.GenLoginParam(loginSchema.Token) + if err != nil { + return nil, fmt.Errorf("failed to setup login parameter: %w", err) + } + jsonData, err := json.Marshal(param) if err != nil { return nil, fmt.Errorf("failed to marshal login parameter: %w", err) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 302292fb16..399a424fe1 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -56,10 +56,14 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client } func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { - return newUpClient(cliMgr.cfg) + return newUpClient(cliMgr.cfg, cliMgr) } -func (cliMgr *ClientManager) generateLoginParameter(privKey []byte, challenge string) (*types.LoginParameter, error) { +func (cliMgr *ClientManager) OnError(isUnauth bool) { + +} + +func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { // Generate public key string publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) From 7b3a65b35b029e93c8622ef0c163168a4f0bb796 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 22:41:17 +0900 Subject: [PATCH 09/26] framework for auto login --- .../controller/proxy/client_manager.go | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 399a424fe1..6893f34dd4 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -1,10 +1,11 @@ package proxy import ( - "context" "crypto/ecdsa" "fmt" + "sync" + "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" @@ -13,13 +14,18 @@ import ( ) type Client interface { - Client(context.Context) *upClient + Client(*gin.Context) *upClient } type ClientManager struct { cliCfg *config.ProxyClient cfg *config.UpStream privKey *ecdsa.PrivateKey + + cachedCli struct { + sync.RWMutex + cli *upClient + } } // transformToValidPrivateKey safely transforms arbitrary bytes into valid private key bytes @@ -55,8 +61,26 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client }, nil } -func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { - return newUpClient(cliMgr.cfg, cliMgr) +func (cliMgr *ClientManager) doLogin() *upClient { + loginCli := newUpClient(cliMgr.cfg, cliMgr) + + return loginCli +} + +func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { + cliMgr.cachedCli.RLock() + if cliMgr.cachedCli.cli != nil { + defer cliMgr.cachedCli.RUnlock() + return cliMgr.cachedCli.cli + } + cliMgr.cachedCli.RUnlock() + cliMgr.cachedCli.Lock() + defer cliMgr.cachedCli.Unlock() + if cliMgr.cachedCli.cli != nil { + return cliMgr.cachedCli.cli + } + + return nil } func (cliMgr *ClientManager) OnError(isUnauth bool) { From 4f878d9231404310a8b7519695747599492807bf Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 24 Aug 2025 23:05:56 +0900 Subject: [PATCH 10/26] AI step --- .../controller/proxy/client_manager.go | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 6893f34dd4..b22b9847d5 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -1,6 +1,7 @@ package proxy import ( + "context" "crypto/ecdsa" "fmt" "sync" @@ -24,7 +25,9 @@ type ClientManager struct { cachedCli struct { sync.RWMutex - cli *upClient + cli *upClient + completionCtx context.Context + completionDone context.CancelFunc } } @@ -74,13 +77,49 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { return cliMgr.cachedCli.cli } cliMgr.cachedCli.RUnlock() + cliMgr.cachedCli.Lock() - defer cliMgr.cachedCli.Unlock() if cliMgr.cachedCli.cli != nil { + defer cliMgr.cachedCli.Unlock() return cliMgr.cachedCli.cli } - return nil + var completionCtx context.Context + // Check if completion context is set + if cliMgr.cachedCli.completionCtx != nil { + completionCtx = cliMgr.cachedCli.completionCtx + } else { + // Set new completion context and launch login goroutine + ctx, completionDone := context.WithCancel(context.TODO()) + cliMgr.cachedCli.completionCtx = ctx + + // Launch login goroutine + go func() { + defer completionDone() + + loginCli := cliMgr.doLogin() + if loginResult, err := loginCli.Login(context.Background()); err == nil { + loginCli.loginToken = loginResult.Token + + cliMgr.cachedCli.Lock() + cliMgr.cachedCli.cli = loginCli + cliMgr.cachedCli.completionCtx = nil + cliMgr.cachedCli.Unlock() + } + }() + } + cliMgr.cachedCli.Unlock() + + // Wait for completion or request cancellation + select { + case <-ctx.Done(): + return nil + case <-completionCtx.Done(): + cliMgr.cachedCli.Lock() + cli := cliMgr.cachedCli.cli + cliMgr.cachedCli.Unlock() + return cli + } } func (cliMgr *ClientManager) OnError(isUnauth bool) { From 624a7a29b843d89a8643daa246a224670b958e74 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 09:35:10 +0900 Subject: [PATCH 11/26] WIP: AI step --- coordinator/conf/config_proxy.json | 3 +- coordinator/internal/config/proxy_config.go | 6 +- .../internal/controller/proxy/client.go | 2 +- .../controller/proxy/client_manager.go | 116 ++++++++++++++---- 4 files changed, 98 insertions(+), 29 deletions(-) diff --git a/coordinator/conf/config_proxy.json b/coordinator/conf/config_proxy.json index 39c3ca0a5f..886c10bf51 100644 --- a/coordinator/conf/config_proxy.json +++ b/coordinator/conf/config_proxy.json @@ -1,7 +1,8 @@ { "proxy_manager": { "proxy_cli": { - "proxy_name": "proxy_name" + "proxy_name": "proxy_name", + "secret": "client private key" }, "auth": { "secret": "proxy secret key", diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index 5edb7fc783..e2d510a29b 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -17,8 +17,8 @@ type ProxyManager struct { } func (m *ProxyManager) Normalize() { - if m.Client.Auth == nil { - m.Client.Auth = m.Auth + if m.Client.Secret == "" { + m.Client.Secret = m.Auth.Secret } if m.Client.ProxyVersion == "" { @@ -30,7 +30,7 @@ func (m *ProxyManager) Normalize() { type ProxyClient struct { ProxyName string `json:"proxy_name"` ProxyVersion string `json:"proxy_version,omitempty"` - Auth *Auth `json:"auth,omitempty"` + Secret string `json:"secret,omitempty"` } // Coordinator configuration diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 8850274f1c..92b0e5d3c8 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -16,7 +16,7 @@ import ( type ClientHelper interface { GenLoginParam(string) (*types.LoginParameter, error) - OnError(isUnauth bool) + OnResp(*upClient, *http.Response) } // Client wraps an http client with a preset host for coordinator API calls diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index b22b9847d5..bae8954998 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -4,18 +4,20 @@ import ( "context" "crypto/ecdsa" "fmt" + "net/http" "sync" + "time" - "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" + "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) type Client interface { - Client(*gin.Context) *upClient + Client(context.Context) *upClient } type ClientManager struct { @@ -25,9 +27,9 @@ type ClientManager struct { cachedCli struct { sync.RWMutex - cli *upClient - completionCtx context.Context - completionDone context.CancelFunc + cli *upClient + completionCtx context.Context + resultChan chan *upClient } } @@ -52,7 +54,7 @@ func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { - privKey, err := buildPrivateKey([]byte(cliCfg.Auth.Secret)) + privKey, err := buildPrivateKey([]byte(cliCfg.Secret)) if err != nil { return nil, err } @@ -64,13 +66,35 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client }, nil } -func (cliMgr *ClientManager) doLogin() *upClient { - loginCli := newUpClient(cliMgr.cfg, cliMgr) +func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) time.Time { + // Calculate wait time between 2 seconds and cfg.RetryWaitTime + minWait := 2 * time.Second + waitDuration := time.Duration(cliMgr.cfg.RetryWaitTime) * time.Second + if waitDuration < minWait { + waitDuration = minWait + } - return loginCli + for { + log.Info("attempting login to upstream coordinator", "baseURL", cliMgr.cfg.BaseUrl) + loginResult, err := loginCli.Login(ctx) + if err == nil && loginResult != nil { + log.Info("login to upstream coordinator successful", "baseURL", cliMgr.cfg.BaseUrl, "time", loginResult.Time) + return loginResult.Time + } + log.Info("login to upstream coordinator failed, retrying", "baseURL", cliMgr.cfg.BaseUrl, "error", err, "waitDuration", waitDuration) + + timer := time.NewTimer(waitDuration) + select { + case <-ctx.Done(): + timer.Stop() + return time.Now() + case <-timer.C: + // Continue to next retry + } + } } -func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { +func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.RLock() if cliMgr.cachedCli.cli != nil { defer cliMgr.cachedCli.RUnlock() @@ -91,21 +115,54 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { } else { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) - cliMgr.cachedCli.completionCtx = ctx + loginCli := newUpClient(cliMgr.cfg, cliMgr) + cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) // Launch login goroutine go func() { defer completionDone() + expiredT := cliMgr.doLogin(context.Background(), loginCli) + + cliMgr.cachedCli.Lock() + cliMgr.cachedCli.cli = loginCli + cliMgr.cachedCli.completionCtx = nil + + // Launch waiting thread to clear cached client before expiration + go func() { + now := time.Now() + clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration + + // If clear time is too soon (less than 10s from now), set it to 10s from now + if clearTime.Before(now.Add(10 * time.Second)) { + clearTime = now.Add(10 * time.Second) + log.Error("token expiration time is too close, delaying clear time", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT, + "adjustedClearTime", clearTime) + } + + waitDuration := time.Until(clearTime) + log.Info("token expiration monitor started", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT, + "clearTime", clearTime, + "waitDuration", waitDuration) + + timer := time.NewTimer(waitDuration) + select { + case <-ctx.Done(): + timer.Stop() + log.Info("token expiration monitor cancelled", "baseURL", cliMgr.cfg.BaseUrl) + case <-timer.C: + log.Info("clearing cached client before token expiration", + "baseURL", cliMgr.cfg.BaseUrl, + "expiredT", expiredT) + cliMgr.clearCachedCli(loginCli) + } + }() + + cliMgr.cachedCli.Unlock() - loginCli := cliMgr.doLogin() - if loginResult, err := loginCli.Login(context.Background()); err == nil { - loginCli.loginToken = loginResult.Token - - cliMgr.cachedCli.Lock() - cliMgr.cachedCli.cli = loginCli - cliMgr.cachedCli.completionCtx = nil - cliMgr.cachedCli.Unlock() - } }() } cliMgr.cachedCli.Unlock() @@ -115,15 +172,26 @@ func (cliMgr *ClientManager) Client(ctx *gin.Context) *upClient { case <-ctx.Done(): return nil case <-completionCtx.Done(): - cliMgr.cachedCli.Lock() - cli := cliMgr.cachedCli.cli - cliMgr.cachedCli.Unlock() + cli := completionCtx.Value("cli").(*upClient) return cli } } -func (cliMgr *ClientManager) OnError(isUnauth bool) { +func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { + cliMgr.cachedCli.Lock() + if cliMgr.cachedCli.cli == cli { + cliMgr.cachedCli.cli = nil + cliMgr.cachedCli.completionCtx = nil + log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + } + cliMgr.cachedCli.Unlock() +} +func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { + if resp.StatusCode == http.StatusForbidden { + log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + cliMgr.clearCachedCli(cli) + } } func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { From 321dd43af8c0d6e74734cbaf676b31d571bacc8b Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 11:43:50 +0900 Subject: [PATCH 12/26] unit test for client --- coordinator/test/proxy_test.go | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 coordinator/test/proxy_test.go diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go new file mode 100644 index 0000000000..b287e41d8b --- /dev/null +++ b/coordinator/test/proxy_test.go @@ -0,0 +1,69 @@ +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/proxy" +) + +func testProxyClientCfg() *config.ProxyClient { + + return &config.ProxyClient{ + Secret: "test-secret-key", + ProxyName: "test-proxy", + } +} + +func testProxyUpStreamCfg(coordinatorURL string) *config.UpStream { + + return &config.UpStream{ + BaseUrl: fmt.Sprintf("http://%s", coordinatorURL), + RetryWaitTime: 3, + ConnectionTimeoutSec: 30, + } + +} + +func testProxyClient(t *testing.T) { + + // Setup coordinator and http server. + coordinatorURL := randomURL() + proofCollector, httpHandler := setupCoordinator(t, 1, coordinatorURL) + defer func() { + proofCollector.Stop() + assert.NoError(t, httpHandler.Shutdown(context.Background())) + }() + + cliCfg := testProxyClientCfg() + upCfg := testProxyUpStreamCfg(coordinatorURL) + + clientManager, err := proxy.NewClientManager(cliCfg, upCfg) + assert.NoError(t, err) + assert.NotNil(t, clientManager) + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Test Client method + client := clientManager.Client(ctx) + + // Client should not be nil if login succeeds + // Note: This might be nil if the coordinator is not properly set up for proxy authentication + // but the test validates that the Client method completes without panic + t.Logf("Client toke: %v", client) + +} + +func TestProxyClient(t *testing.T) { + + // Set up the test environment. + setEnv(t) + t.Run("TestProxyHandshake", testProxyClient) +} From 64ef0f4ec038c1a8a41950a408bc3228cfde7a46 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 25 Aug 2025 11:52:03 +0900 Subject: [PATCH 13/26] WIP --- .../controller/proxy/client_manager.go | 22 ++++++++++--------- .../internal/controller/proxy/controller.go | 2 +- coordinator/test/proxy_test.go | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index bae8954998..ec57a3b351 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -21,6 +21,7 @@ type Client interface { } type ClientManager struct { + name string cliCfg *config.ProxyClient cfg *config.UpStream privKey *ecdsa.PrivateKey @@ -52,7 +53,7 @@ func buildPrivateKey(inputBytes []byte) (*ecdsa.PrivateKey, error) { return nil, fmt.Errorf("failed to generate valid private key from input bytes") } -func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { +func NewClientManager(name string, cliCfg *config.ProxyClient, cfg *config.UpStream) (*ClientManager, error) { privKey, err := buildPrivateKey([]byte(cliCfg.Secret)) if err != nil { @@ -60,6 +61,7 @@ func NewClientManager(cliCfg *config.ProxyClient, cfg *config.UpStream) (*Client } return &ClientManager{ + name: name, privKey: privKey, cfg: cfg, cliCfg: cliCfg, @@ -75,13 +77,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } for { - log.Info("attempting login to upstream coordinator", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("attempting login to upstream coordinator", "name", cliMgr.name) loginResult, err := loginCli.Login(ctx) if err == nil && loginResult != nil { - log.Info("login to upstream coordinator successful", "baseURL", cliMgr.cfg.BaseUrl, "time", loginResult.Time) + log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) return loginResult.Time } - log.Info("login to upstream coordinator failed, retrying", "baseURL", cliMgr.cfg.BaseUrl, "error", err, "waitDuration", waitDuration) + log.Info("login to upstream coordinator failed, retrying", "name", cliMgr.name, "error", err, "waitDuration", waitDuration) timer := time.NewTimer(waitDuration) select { @@ -136,14 +138,14 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { if clearTime.Before(now.Add(10 * time.Second)) { clearTime = now.Add(10 * time.Second) log.Error("token expiration time is too close, delaying clear time", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT, "adjustedClearTime", clearTime) } waitDuration := time.Until(clearTime) log.Info("token expiration monitor started", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT, "clearTime", clearTime, "waitDuration", waitDuration) @@ -152,10 +154,10 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { select { case <-ctx.Done(): timer.Stop() - log.Info("token expiration monitor cancelled", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("token expiration monitor cancelled", "name", cliMgr.name) case <-timer.C: log.Info("clearing cached client before token expiration", - "baseURL", cliMgr.cfg.BaseUrl, + "name", cliMgr.name, "expiredT", expiredT) cliMgr.clearCachedCli(loginCli) } @@ -182,14 +184,14 @@ func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { if cliMgr.cachedCli.cli == cli { cliMgr.cachedCli.cli = nil cliMgr.cachedCli.completionCtx = nil - log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) } cliMgr.cachedCli.Unlock() } func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { if resp.StatusCode == http.StatusForbidden { - log.Info("cached client cleared due to forbidden response", "baseURL", cliMgr.cfg.BaseUrl) + log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) cliMgr.clearCachedCli(cli) } } diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 297f8c68a4..5c42927685 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -35,7 +35,7 @@ func InitController(cfg *config.ProxyConfig) { clients := make(map[string]Client) for nm, upCfg := range cfg.Coordinators { - cli, err := NewClientManager(cfg.ProxyManager.Client, upCfg) + cli, err := NewClientManager(nm, cfg.ProxyManager.Client, upCfg) if err != nil { panic("create new client fail") } diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index b287e41d8b..670d6548ca 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -43,7 +43,7 @@ func testProxyClient(t *testing.T) { cliCfg := testProxyClientCfg() upCfg := testProxyUpStreamCfg(coordinatorURL) - clientManager, err := proxy.NewClientManager(cliCfg, upCfg) + clientManager, err := proxy.NewClientManager("test_coordinator", cliCfg, upCfg) assert.NoError(t, err) assert.NotNil(t, clientManager) From 5a07a1652bebb8bf450b12a08891e89fb116592b Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 27 Aug 2025 09:43:30 +0900 Subject: [PATCH 14/26] WIP --- coordinator/internal/controller/proxy/auth.go | 46 +++++++++++++++++++ .../internal/controller/proxy/get_task.go | 45 ++++++++++++++++++ .../internal/controller/proxy/submit_proof.go | 29 ++++++++++++ .../internal/middleware/proxy_bearer.go | 1 + 4 files changed, 121 insertions(+) create mode 100644 coordinator/internal/controller/proxy/auth.go create mode 100644 coordinator/internal/controller/proxy/get_task.go create mode 100644 coordinator/internal/controller/proxy/submit_proof.go create mode 100644 coordinator/internal/middleware/proxy_bearer.go diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go new file mode 100644 index 0000000000..97fffc8c76 --- /dev/null +++ b/coordinator/internal/controller/proxy/auth.go @@ -0,0 +1,46 @@ +package proxy + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/logic/auth" + "scroll-tech/coordinator/internal/logic/verifier" + "scroll-tech/coordinator/internal/types" +) + +// AuthController is login API +type AuthController struct { + *api.AuthController + clients Clients +} + +// NewAuthController returns an LoginController instance +func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { + + loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) + auth := api.NewAuthControllerWithLogic(loginLogic) + return &AuthController{ + AuthController: auth, + clients: clients, + } +} + +// Login extended the Login hander in api controller +func (a *AuthController) Login(c *gin.Context) (interface{}, error) { + + ret, err := a.AuthController.Login(c) + if err != nil { + return nil, err + } + loginParam := ret.(types.LoginParameterWithHardForkName) + // band recursive proxy now ... + if loginParam.Message.ProverProviderType == types.ProverProviderTypeProxy { + return nil, fmt.Errorf("do not allow recursive proxy for login %v", loginParam.Message) + } + + return loginParam, nil +} diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go new file mode 100644 index 0000000000..f351389a24 --- /dev/null +++ b/coordinator/internal/controller/proxy/get_task.go @@ -0,0 +1,45 @@ +package proxy + +import ( + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/params" + "gorm.io/gorm" + + "scroll-tech/common/types/message" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/provertask" + "scroll-tech/coordinator/internal/logic/verifier" + coordinatorType "scroll-tech/coordinator/internal/types" +) + +// GetTaskController the get prover task api controller +type GetTaskController struct { + proverTasks map[message.ProofType]provertask.ProverTask + + getTaskAccessCounter *prometheus.CounterVec +} + +// NewGetTaskController create a get prover task controller +func NewGetTaskController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, verifier *verifier.Verifier, reg prometheus.Registerer) *GetTaskController { + // TODO: implement proxy get task controller initialization + return &GetTaskController{ + proverTasks: make(map[message.ProofType]provertask.ProverTask), + } +} + +func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { + // TODO: implement proxy get task access counter + return nil +} + +// GetTasks get assigned chunk/batch task +func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { + // TODO: implement proxy get tasks logic +} + +func (ptc *GetTaskController) proofType(para *coordinatorType.GetTaskParameter) message.ProofType { + // TODO: implement proxy proof type logic + return message.ProofTypeChunk +} \ No newline at end of file diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go new file mode 100644 index 0000000000..f2afa12945 --- /dev/null +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -0,0 +1,29 @@ +package proxy + +import ( + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/scroll-tech/go-ethereum/params" + "gorm.io/gorm" + + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/logic/submitproof" + "scroll-tech/coordinator/internal/logic/verifier" +) + +// SubmitProofController the submit proof api controller +type SubmitProofController struct { + submitProofReceiverLogic *submitproof.ProofReceiverLogic +} + +// NewSubmitProofController create the submit proof api controller instance +func NewSubmitProofController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, vf *verifier.Verifier, reg prometheus.Registerer) *SubmitProofController { + return &SubmitProofController{ + submitProofReceiverLogic: submitproof.NewSubmitProofReceiverLogic(cfg.ProverManager, chainCfg, db, vf, reg), + } +} + +// SubmitProof prover submit the proof to coordinator +func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { + // TODO: implement proxy submit proof logic +} \ No newline at end of file diff --git a/coordinator/internal/middleware/proxy_bearer.go b/coordinator/internal/middleware/proxy_bearer.go new file mode 100644 index 0000000000..c870d7c164 --- /dev/null +++ b/coordinator/internal/middleware/proxy_bearer.go @@ -0,0 +1 @@ +package middleware From 5614ec3b8633f45c19885fccc5b0841ebe026976 Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 1 Sep 2025 10:12:16 +0900 Subject: [PATCH 15/26] WIP --- coordinator/internal/controller/api/auth.go | 14 +- coordinator/internal/controller/proxy/auth.go | 316 +++++++++++++++++- .../internal/controller/proxy/client.go | 29 +- .../controller/proxy/client_manager.go | 1 - coordinator/internal/middleware/login_jwt.go | 9 + 5 files changed, 346 insertions(+), 23 deletions(-) diff --git a/coordinator/internal/controller/api/auth.go b/coordinator/internal/controller/api/auth.go index 1a3305d2b1..c2abb86f09 100644 --- a/coordinator/internal/controller/api/auth.go +++ b/coordinator/internal/controller/api/auth.go @@ -19,6 +19,12 @@ type AuthController struct { loginLogic *auth.LoginLogic } +func NewAuthControllerWithLogic(loginLogic *auth.LoginLogic) *AuthController { + return &AuthController{ + loginLogic: loginLogic, + } +} + // NewAuthController returns an LoginController instance func NewAuthController(db *gorm.DB, cfg *config.Config, vf *verifier.Verifier) *AuthController { return &AuthController{ @@ -102,10 +108,6 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { c.Set(types.ProverName, proverName) } - if publicKey, ok := claims[types.PublicKey]; ok { - c.Set(types.PublicKey, publicKey) - } - if proverVersion, ok := claims[types.ProverVersion]; ok { c.Set(types.ProverVersion, proverVersion) } @@ -118,5 +120,9 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { c.Set(types.ProverProviderTypeKey, providerType) } + if publicKey, ok := claims[types.PublicKey]; ok { + return publicKey + } + return nil } diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 97fffc8c76..5c293fae3a 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -2,8 +2,14 @@ package proxy import ( "fmt" + "sync" + "context" + "time" + + jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" + "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" @@ -14,33 +20,319 @@ import ( // AuthController is login API type AuthController struct { - *api.AuthController - clients Clients + apiLogin *api.AuthController + clients Clients + userTokenCache *UserTokenCache + tokenCacheUpdate chan<- *TokenUpdate +} + +type TokenUpdate struct { + PublicKey string + Upstream string + Phase uint + LoginParam types.LoginParameter + CompleteNotify chan<- *types.LoginSchema +} + +type UpstreamTokens struct { + LoginData map[string]*types.LoginSchema + LoginPhase uint + NextLoginPhase uint +} + +type UserTokenCache struct { + sync.RWMutex + data map[string]UpstreamTokens +} + +func newUserTokens() UpstreamTokens { + return UpstreamTokens{ + LoginData: make(map[string]*types.LoginSchema), + } +} + +func newUserCache() *UserTokenCache { + return &UserTokenCache{data: make(map[string]UpstreamTokens)} +} + +// get retrieves UpstreamTokens for a given user key, returns empty if still not exists +func (c *UserTokenCache) Get(userKey string) *UpstreamTokens { + c.RLock() + defer c.RUnlock() + + tokens, exists := c.data[userKey] + if !exists { + return nil + } + + return &tokens +} + +// prepare for a total update via Login request +func (c *UserTokenCache) updatePrepare(userKey string) UpstreamTokens { + c.Lock() + defer c.Unlock() + + if _, exists := c.data[userKey]; !exists { + log.Info("initializing user token cache", "userKey", userKey) + c.data[userKey] = newUserTokens() + } + updated := c.data[userKey] + updated.NextLoginPhase = updated.LoginPhase + 1 + c.data[userKey] = updated + return updated } +// partialSet updates a single entry in upstreamTokens for a given user +func (c *UserTokenCache) partialSet(userKey string, upstreamName string, loginSchema *types.LoginSchema, phase uint) { + c.Lock() + defer c.Unlock() + + // Get existing tokens or create new map + tokens, exists := c.data[userKey] + if exists && tokens.NextLoginPhase == phase { + // Update the specific upstream entry + tokens.LoginData[upstreamName] = loginSchema + } +} + +// LoginParameterWithHardForkName constructs new payload for login +type LoginParameterWithUpstreamTokens struct { + *types.LoginParameter + Tokens UpstreamTokens +} + +const upstreamConnTimeout = time.Second * 2 +const expireTolerant = 10 * time.Minute +const LoginParamCache = "login_param" +const ProverTypesKey = "prover_types" +const SignatureKey = "prover_signature" + // NewAuthController returns an LoginController instance func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) - auth := api.NewAuthControllerWithLogic(loginLogic) - return &AuthController{ - AuthController: auth, - clients: clients, + + // Create the token cache update channel + tokenCacheUpdateChan := make(chan *TokenUpdate) + + authController := &AuthController{ + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + userTokenCache: newUserCache(), + tokenCacheUpdate: tokenCacheUpdateChan, + } + + // Launch token cache manager in a separate goroutine + go authController.toeknCacheManager(tokenCacheUpdateChan) + + return authController +} + +func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { + if req.CompleteNotify != nil { + defer func(ctx context.Context) { + select { + case <-ctx.Done(): + case req.CompleteNotify <- ret: + } + + }(ctx) } + + cli := a.clients[req.Upstream] + if cli := cli.Client(ctx); cli != nil { + var err error + if ret, err = cli.ProxyLogin(ctx, req.LoginParam); err == nil { + a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) + } else { + log.Error("proxy login failed during token cache update", + "userKey", req.PublicKey, + "upstream", req.Upstream, + "phase", req.Phase, + "error", err) + } + } + return + +} + +func (a *AuthController) toeknCacheManager(request <-chan *TokenUpdate) { + + ctx := context.TODO() + var managerStatusLock sync.Mutex + managerStatus := make(map[string]map[string]uint) + + for { + req, ok := <-request + if !ok { + return + } + + // ensure the manager request is not outdated + tokens := a.userTokenCache.Get(req.PublicKey) + if tokens == nil { + // Highly not possible, if raise, the reason is unknown, just log the Error + continue + } + phase := tokens.NextLoginPhase + if req.Phase < phase { + // drop the out-dated request + continue + } + + // ensure only one login request is launched for the same phase + managerStatusLock.Lock() + stat, ok := managerStatus[req.Upstream] + if !ok { + managerStatus[req.Upstream] = make(map[string]uint) + stat = managerStatus[req.Upstream] + } + if phase, running := stat[req.PublicKey]; running && phase >= req.Phase { + managerStatusLock.Unlock() + continue + } else { + stat[req.PublicKey] = req.Phase + } + managerStatusLock.Unlock() + + go a.doUpdateRequest(ctx, req) + + } + } // Login extended the Login hander in api controller func (a *AuthController) Login(c *gin.Context) (interface{}, error) { - ret, err := a.AuthController.Login(c) + loginRes, err := a.apiLogin.Login(c) if err != nil { return nil, err } - loginParam := ret.(types.LoginParameterWithHardForkName) - // band recursive proxy now ... - if loginParam.Message.ProverProviderType == types.ProverProviderTypeProxy { - return nil, fmt.Errorf("do not allow recursive proxy for login %v", loginParam.Message) + loginParam := loginRes.(types.LoginParameterWithHardForkName) + + if loginParam.LoginParameter.Message.ProverProviderType == types.ProverProviderTypeProxy { + return nil, fmt.Errorf("proxy do not support recursive login") + } + + tokens := a.userTokenCache.updatePrepare(loginParam.PublicKey) + notifies := make([]chan *types.LoginSchema, len(a.clients)) + + for n := range a.clients { + + // Check if we have a valid cached token that hasn't expired + if knownEntry, existed := tokens.LoginData[n]; existed { + timeRemaining := time.Until(knownEntry.Time) + if timeRemaining > expireTolerant { + // Token is still valid enouth, continue to next client + continue + } + } + + notify := make(chan *types.LoginSchema) + notifies = append(notifies, notify) + request := TokenUpdate{ + PublicKey: loginParam.PublicKey, + Upstream: n, + Phase: tokens.NextLoginPhase, + LoginParam: loginParam.LoginParameter, + CompleteNotify: notify, + } + defer close(notify) + select { + case <-c.Done(): + case a.tokenCacheUpdate <- &request: + } + + } + + // collect all request's compeletions + for _, chn := range notifies { + select { + case <-c.Done(): + case <-chn: + } + } + + return LoginParameterWithUpstreamTokens{ + LoginParameter: &loginParam.LoginParameter, + Tokens: tokens, + }, nil +} + +// PayloadFunc returns jwt.MapClaims with {public key, prover name}. +func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims { + v, ok := data.(LoginParameterWithUpstreamTokens) + if !ok { + return jwt.MapClaims{} + } + + return jwt.MapClaims{ + types.PublicKey: v.PublicKey, + types.ProverName: v.Message.ProverName, + types.ProverVersion: v.Message.ProverVersion, + types.ProverProviderTypeKey: v.Message.ProverProviderType, + SignatureKey: v.Signature, + ProverTypesKey: v.Message.ProverTypes, + } +} + +// IdentityHandler replies to client for /login +func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { + claims := jwt.ExtractClaims(c) + loginParam := &types.LoginParameter{} + + if proverName, ok := claims[types.ProverName]; ok { + loginParam.Message.ProverName, _ = proverName.(string) + } + + if proverVersion, ok := claims[types.ProverVersion]; ok { + loginParam.Message.ProverVersion, _ = proverVersion.(string) + } + + if providerType, ok := claims[types.ProverProviderTypeKey]; ok { + num, _ := providerType.(float64) + loginParam.Message.ProverProviderType = types.ProverProviderType(num) + } + + if signature, ok := claims[SignatureKey]; ok { + loginParam.Signature, _ = signature.(string) + } + + if proverTypes, ok := claims[ProverTypesKey]; ok { + arr, _ := proverTypes.([]any) + for _, elm := range arr { + num, _ := elm.(float64) + loginParam.Message.ProverTypes = append(loginParam.Message.ProverTypes, types.ProverType(num)) + } + } + + if publicKey, ok := claims[types.PublicKey]; ok { + loginParam.PublicKey, _ = publicKey.(string) + } + + if loginParam.PublicKey != "" { + // ensure tokenCache + a.userTokenCache.RLock() + _, exists := a.userTokenCache.data[loginParam.PublicKey] + if !exists { + a.userTokenCache.RUnlock() + a.userTokenCache.Lock() + if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { + log.Info("creating token cache for user after proxy restart", + "publicKey", loginParam.PublicKey, + "proverName", loginParam.Message.ProverName, + "reason", "prover using JWT token from before proxy restart") + a.userTokenCache.data[loginParam.PublicKey] = newUserTokens() + } + a.userTokenCache.Unlock() + } else { + a.userTokenCache.RUnlock() + } + + c.Set(LoginParamCache, loginParam) + return loginParam.PublicKey } - return loginParam, nil + return nil } diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 92b0e5d3c8..29c78cf216 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -8,8 +8,6 @@ import ( "net/http" "time" - "github.com/gin-gonic/gin" - "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" ) @@ -104,7 +102,7 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*http.Response, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -120,11 +118,30 @@ func (c *upClient) ProxyLogin(ctx *gin.Context, param types.LoginParameter) (*ht req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+c.loginToken) - return c.httpClient.Do(req) + proxyLoginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform proxy login request: %w", err) + } + defer proxyLoginResp.Body.Close() + + // Call helper's OnResp method with the response + c.helper.OnResp(c, proxyLoginResp) + + // Parse proxy login response as LoginSchema + if proxyLoginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { + return &loginResult, nil + } + // If parsing fails, still return success but with nil result + return nil, nil + } + + return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -146,7 +163,7 @@ func (c *upClient) GetTask(ctx *gin.Context, param types.GetTaskParameter, token } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx *gin.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index ec57a3b351..31e0a324cf 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -30,7 +30,6 @@ type ClientManager struct { sync.RWMutex cli *upClient completionCtx context.Context - resultChan chan *upClient } } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index b04810b0b7..565aff9daf 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -4,6 +4,7 @@ import ( "time" jwt "github.com/appleboy/gin-jwt/v2" + "github.com/gin-gonic/gin" "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" @@ -11,6 +12,13 @@ import ( "scroll-tech/coordinator/internal/types" ) +func nonIdendityAuthorizator(data interface{}, _ *gin.Context) bool { + if data == nil { + return false + } + return true +} + // LoginMiddleware jwt auth middleware func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ @@ -20,6 +28,7 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { Key: []byte(conf.Auth.Secret), Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), Authenticator: api.Auth.Login, + Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, TokenLookup: "header: Authorization, query: token, cookie: jwt", TokenHeadName: "Bearer", From 322766f54f8432452336454eb3fb84bea38e23c2 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 2 Sep 2025 17:22:35 +0900 Subject: [PATCH 16/26] WIP --- coordinator/internal/controller/proxy/auth.go | 40 ++--- .../internal/controller/proxy/client.go | 24 ++- .../controller/proxy/client_manager.go | 8 + .../internal/controller/proxy/get_task.go | 142 ++++++++++++++++-- 4 files changed, 176 insertions(+), 38 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 5c293fae3a..2fc1d76aa0 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -20,10 +20,9 @@ import ( // AuthController is login API type AuthController struct { - apiLogin *api.AuthController - clients Clients - userTokenCache *UserTokenCache - tokenCacheUpdate chan<- *TokenUpdate + apiLogin *api.AuthController + clients Clients + userTokenCache *UserTokenCache } type TokenUpdate struct { @@ -42,7 +41,8 @@ type UpstreamTokens struct { type UserTokenCache struct { sync.RWMutex - data map[string]UpstreamTokens + data map[string]UpstreamTokens + tokenCacheUpdate chan<- *TokenUpdate } func newUserTokens() UpstreamTokens { @@ -51,8 +51,11 @@ func newUserTokens() UpstreamTokens { } } -func newUserCache() *UserTokenCache { - return &UserTokenCache{data: make(map[string]UpstreamTokens)} +func newUserCache(tokenCacheUpdate chan<- *TokenUpdate) *UserTokenCache { + return &UserTokenCache{ + data: make(map[string]UpstreamTokens), + tokenCacheUpdate: tokenCacheUpdate, + } } // get retrieves UpstreamTokens for a given user key, returns empty if still not exists @@ -117,10 +120,9 @@ func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Ve tokenCacheUpdateChan := make(chan *TokenUpdate) authController := &AuthController{ - apiLogin: api.NewAuthControllerWithLogic(loginLogic), - clients: clients, - userTokenCache: newUserCache(), - tokenCacheUpdate: tokenCacheUpdateChan, + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + userTokenCache: newUserCache(tokenCacheUpdateChan), } // Launch token cache manager in a separate goroutine @@ -129,6 +131,8 @@ func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Ve return authController } +func (a *AuthController) TokenCache() *UserTokenCache { return a.userTokenCache } + func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { if req.CompleteNotify != nil { defer func(ctx context.Context) { @@ -143,13 +147,13 @@ func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) cli := a.clients[req.Upstream] if cli := cli.Client(ctx); cli != nil { var err error - if ret, err = cli.ProxyLogin(ctx, req.LoginParam); err == nil { + if ret, err = cli.ProxyLogin(ctx, &req.LoginParam); err == nil { a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) } else { - log.Error("proxy login failed during token cache update", - "userKey", req.PublicKey, - "upstream", req.Upstream, - "phase", req.Phase, + log.Error("proxy login failed during token cache update", + "userKey", req.PublicKey, + "upstream", req.Upstream, + "phase", req.Phase, "error", err) } } @@ -241,7 +245,7 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { defer close(notify) select { case <-c.Done(): - case a.tokenCacheUpdate <- &request: + case a.userTokenCache.tokenCacheUpdate <- &request: } } @@ -319,7 +323,7 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { a.userTokenCache.RUnlock() a.userTokenCache.Lock() if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { - log.Info("creating token cache for user after proxy restart", + log.Info("creating token cache for user after proxy restart", "publicKey", loginParam.PublicKey, "proverName", loginParam.Message.ProverName, "reason", "prover using JWT token from before proxy restart") diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 29c78cf216..549fbc3670 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -8,8 +8,11 @@ import ( "net/http" "time" + ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" + + "github.com/mitchellh/mapstructure" ) type ClientHelper interface { @@ -90,19 +93,26 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Parse login response as LoginSchema and store the token if loginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(loginResp.Body).Decode(&loginResult); err == nil { + var respWithData ctypes.Response + // Note: Body is consumed after decoding, caller should not read it again + if err := json.NewDecoder(loginResp.Body).Decode(&respWithData); err == nil { + var loginResult types.LoginSchema + err = mapstructure.Decode(respWithData.Data, &loginResult) + if err != nil { + return nil, fmt.Errorf("login parsing data fail, get %v", respWithData.Data) + } c.loginToken = loginResult.Token + return &loginResult, nil + } else { + return nil, fmt.Errorf("login parsing response failed: %v", err) } - // Note: Body is consumed after decoding, caller should not read it again - return &loginResult, nil } return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*types.LoginSchema, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -141,7 +151,7 @@ func (c *upClient) ProxyLogin(ctx context.Context, param types.LoginParameter) ( } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -163,7 +173,7 @@ func (c *upClient) GetTask(ctx context.Context, param types.GetTaskParameter, to } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx context.Context, param types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 31e0a324cf..e090c3ee73 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -18,6 +18,7 @@ import ( type Client interface { Client(context.Context) *upClient + PeekClient() *upClient } type ClientManager struct { @@ -95,6 +96,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } } +func (cliMgr *ClientManager) PeekClient() *upClient { + cliMgr.cachedCli.RLock() + defer cliMgr.cachedCli.RUnlock() + + return cliMgr.cachedCli.cli +} + func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.RLock() if cliMgr.cachedCli.cli != nil { diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index f351389a24..261219aa2f 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -1,31 +1,62 @@ package proxy import ( + "encoding/json" + "fmt" + "net/http" + "github.com/gin-gonic/gin" + "github.com/mitchellh/mapstructure" "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/params" - "gorm.io/gorm" + "github.com/scroll-tech/go-ethereum/log" - "scroll-tech/common/types/message" + "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/provertask" - "scroll-tech/coordinator/internal/logic/verifier" coordinatorType "scroll-tech/coordinator/internal/types" ) +func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) { + + publicKeyData, publicKeyExist := ctx.Get(coordinatorType.PublicKey) + publicKey, castOk := publicKeyData.(string) + if !publicKeyExist || !castOk { + nerr := fmt.Errorf("no public key binding: %v", publicKeyData) + log.Warn("get_task parameter fail", "error", nerr) + + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return "", nil + } + + loginParamData, publicKeyExist := ctx.Get(LoginParamCache) + loginParam, castOk := loginParamData.(*coordinatorType.LoginParameter) + if !publicKeyExist || !castOk { + nerr := fmt.Errorf("no login param binding: %v", loginParamData) + log.Warn("get_task parameter fail", "error", nerr) + + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return "", nil + } + + return publicKey, loginParam +} + // GetTaskController the get prover task api controller type GetTaskController struct { - proverTasks map[message.ProofType]provertask.ProverTask + tokenCache *UserTokenCache + clients Clients + priorityUpstream map[string]string getTaskAccessCounter *prometheus.CounterVec } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, verifier *verifier.Verifier, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.Config, clients Clients, tokenCache *UserTokenCache, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ - proverTasks: make(map[message.ProofType]provertask.ProverTask), + priorityUpstream: make(map[string]string), + tokenCache: tokenCache, + clients: clients, } } @@ -36,10 +67,95 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { - // TODO: implement proxy get tasks logic + var getTaskParameter coordinatorType.GetTaskParameter + if err := ctx.ShouldBind(&getTaskParameter); err != nil { + nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + publicKey, loginParam := getSessionData(ctx) + if publicKey == "" || loginParam == nil { + return + } + + tokens := ptc.tokenCache.Get(publicKey) + + onClientFail := func(upstream string) { + //TODO: log re-connect request in info level + + request := TokenUpdate{ + PublicKey: publicKey, + Upstream: upstream, + Phase: tokens.LoginPhase, + LoginParam: *loginParam, + CompleteNotify: nil, + } + select { + case <-ctx.Done(): + case ptc.tokenCache.tokenCacheUpdate <- &request: + } + + } + + priorityUpstream, exist := ptc.priorityUpstream[publicKey] + if exist { + cli := ptc.clients[priorityUpstream] + loginSchema := tokens.LoginData[priorityUpstream] + if loginSchema == nil { + onClientFail(priorityUpstream) + } else { + ret, triggerUpdate := getTaskFromClient(ctx, cli, &getTaskParameter, loginSchema.Token) + if ret != nil { + + } else if triggerUpdate { + onClientFail(priorityUpstream) + } + } + types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) + } + + for n, cli := range ptc.clients { + + } } -func (ptc *GetTaskController) proofType(para *coordinatorType.GetTaskParameter) message.ProofType { - // TODO: implement proxy proof type logic - return message.ProofTypeChunk -} \ No newline at end of file +func getTaskFromClient(ctx *gin.Context, cli Client, param *coordinatorType.GetTaskParameter, token string) (*coordinatorType.GetTaskSchema, bool) { + + theCli := cli.PeekClient() + if theCli == nil { + return nil, true + } + + resp, err := theCli.GetTask(ctx, param, token) + if err != nil { + // log the err in error level + return nil, false + } + + // Parse response + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { + unAuth := resp.StatusCode == http.StatusUnauthorized + var respWithData types.Response + // Note: Body is consumed after decoding, caller should not read it again + if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { + if unAuth && respWithData.ErrCode == types.ErrJWTTokenExpired { + return nil, true + } + + var getTaskResult coordinatorType.GetTaskSchema + err = mapstructure.Decode(respWithData.Data, &getTaskResult) + if err != nil { + log.Error("parse get task data fail", "respdata", respWithData.Data) + return nil, false + } + return &getTaskResult, false + } else { + log.Error("parse get task response failed", "error", err) + //fmt.Errorf("login parsing response failed: %v", err) + return nil, false + } + } + + return nil, false +} From e6be62f6333c1c1b4f061cca600c4c3ec3dd06f7 Mon Sep 17 00:00:00 2001 From: Ho Date: Fri, 5 Sep 2025 22:31:45 +0900 Subject: [PATCH 17/26] WIP --- common/types/response.go | 5 + .../internal/controller/proxy/client.go | 86 +++++------ .../controller/proxy/client_manager.go | 105 ++++++------- .../controller/proxy/prover_session.go | 140 ++++++++++++++++++ 4 files changed, 230 insertions(+), 106 deletions(-) create mode 100644 coordinator/internal/controller/proxy/prover_session.go diff --git a/common/types/response.go b/common/types/response.go index d616aa8013..abd29041e7 100644 --- a/common/types/response.go +++ b/common/types/response.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/mitchellh/mapstructure" ) // Response the response schema @@ -13,6 +14,10 @@ type Response struct { Data interface{} `json:"data"` } +func (resp *Response) DecodeData(out interface{}) error { + return mapstructure.Decode(resp.Data, out) +} + // RenderJSON renders response with json func RenderJSON(ctx *gin.Context, errCode int, err error, data interface{}) { var errMsg string diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index 549fbc3670..e49caa83f3 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -11,36 +11,27 @@ import ( ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/types" - - "github.com/mitchellh/mapstructure" ) -type ClientHelper interface { - GenLoginParam(string) (*types.LoginParameter, error) - OnResp(*upClient, *http.Response) -} - // Client wraps an http client with a preset host for coordinator API calls type upClient struct { httpClient *http.Client baseURL string loginToken string - helper ClientHelper } // NewClient creates a new Client with the specified host -func newUpClient(cfg *config.UpStream, helper ClientHelper) *upClient { +func newUpClient(cfg *config.UpStream) *upClient { return &upClient{ httpClient: &http.Client{ Timeout: time.Duration(cfg.ConnectionTimeoutSec) * time.Second, }, baseURL: cfg.BaseUrl, - helper: helper, } } // FullLogin performs the complete login process: get challenge then login -func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { +func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*types.LoginSchema, error) { // Step 1: Get challenge url := fmt.Sprintf("%s/coordinator/v1/challenge", c.baseURL) @@ -68,7 +59,7 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) - param, err := c.helper.GenLoginParam(loginSchema.Token) + param, err := genLogin(loginSchema.Token) if err != nil { return nil, fmt.Errorf("failed to setup login parameter: %w", err) } @@ -91,28 +82,38 @@ func (c *upClient) Login(ctx context.Context) (*types.LoginSchema, error) { return nil, fmt.Errorf("failed to perform login request: %w", err) } - // Parse login response as LoginSchema and store the token - if loginResp.StatusCode == http.StatusOK { + parsedResp, err := handleHttpResp(loginResp) + if err != nil { + return nil, err + } + + var loginResult types.LoginSchema + err = parsedResp.DecodeData(&loginResult) + if err != nil { + return nil, fmt.Errorf("login parsing data fail: %v", err) + } + c.loginToken = loginResult.Token + return &loginResult, nil + +} + +func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { + defer resp.Body.Close() var respWithData ctypes.Response // Note: Body is consumed after decoding, caller should not read it again - if err := json.NewDecoder(loginResp.Body).Decode(&respWithData); err == nil { - var loginResult types.LoginSchema - err = mapstructure.Decode(respWithData.Data, &loginResult) - if err != nil { - return nil, fmt.Errorf("login parsing data fail, get %v", respWithData.Data) - } - c.loginToken = loginResult.Token - return &loginResult, nil + if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { + return &respWithData, nil } else { - return nil, fmt.Errorf("login parsing response failed: %v", err) + return nil, fmt.Errorf("login parsing expected response failed: %v", err) } - } - return nil, fmt.Errorf("login request failed with status: %d", loginResp.StatusCode) + } + return nil, fmt.Errorf("login request failed with status: %d", resp.StatusCode) } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*types.LoginSchema, error) { +func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) jsonData, err := json.Marshal(param) @@ -132,26 +133,11 @@ func (c *upClient) ProxyLogin(ctx context.Context, param *types.LoginParameter) if err != nil { return nil, fmt.Errorf("failed to perform proxy login request: %w", err) } - defer proxyLoginResp.Body.Close() - - // Call helper's OnResp method with the response - c.helper.OnResp(c, proxyLoginResp) - - // Parse proxy login response as LoginSchema - if proxyLoginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { - return &loginResult, nil - } - // If parsing fails, still return success but with nil result - return nil, nil - } - - return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) + return handleHttpResp(proxyLoginResp) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { +func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) jsonData, err := json.Marshal(param) @@ -169,11 +155,15 @@ func (c *upClient) GetTask(ctx context.Context, param *types.GetTaskParameter, t req.Header.Set("Authorization", "Bearer "+token) } - return c.httpClient.Do(req) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + return handleHttpResp(resp) } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { +func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*ctypes.Response, error) { url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) jsonData, err := json.Marshal(param) @@ -191,5 +181,9 @@ func (c *upClient) SubmitProof(ctx context.Context, param *types.SubmitProofPara req.Header.Set("Authorization", "Bearer "+token) } - return c.httpClient.Do(req) + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + return handleHttpResp(resp) } diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index e090c3ee73..968c66b328 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "fmt" - "net/http" "sync" "time" @@ -18,7 +17,7 @@ import ( type Client interface { Client(context.Context) *upClient - PeekClient() *upClient + Reset(cli *upClient) } type ClientManager struct { @@ -78,7 +77,7 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti for { log.Info("attempting login to upstream coordinator", "name", cliMgr.name) - loginResult, err := loginCli.Login(ctx) + loginResult, err := loginCli.Login(ctx, cliMgr.genLoginParam) if err == nil && loginResult != nil { log.Info("login to upstream coordinator successful", "name", cliMgr.name, "time", loginResult.Time) return loginResult.Time @@ -96,11 +95,13 @@ func (cliMgr *ClientManager) doLogin(ctx context.Context, loginCli *upClient) ti } } -func (cliMgr *ClientManager) PeekClient() *upClient { - cliMgr.cachedCli.RLock() - defer cliMgr.cachedCli.RUnlock() - - return cliMgr.cachedCli.cli +func (cliMgr *ClientManager) Reset(cli *upClient) { + cliMgr.cachedCli.Lock() + if cliMgr.cachedCli.cli == cli { + cliMgr.cachedCli.cli = nil + } + cliMgr.cachedCli.Unlock() + log.Info("cached client cleared", "name", cliMgr.name) } func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { @@ -124,51 +125,52 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { } else { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) - loginCli := newUpClient(cliMgr.cfg, cliMgr) + loginCli := newUpClient(cliMgr.cfg) cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) - // Launch login goroutine + // Launch keep-login goroutine go func() { defer completionDone() expiredT := cliMgr.doLogin(context.Background(), loginCli) + log.Info("login compeleted", "name", cliMgr.name, "expired", expiredT) cliMgr.cachedCli.Lock() cliMgr.cachedCli.cli = loginCli cliMgr.cachedCli.completionCtx = nil // Launch waiting thread to clear cached client before expiration - go func() { - now := time.Now() - clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration - - // If clear time is too soon (less than 10s from now), set it to 10s from now - if clearTime.Before(now.Add(10 * time.Second)) { - clearTime = now.Add(10 * time.Second) - log.Error("token expiration time is too close, delaying clear time", - "name", cliMgr.name, - "expiredT", expiredT, - "adjustedClearTime", clearTime) - } - - waitDuration := time.Until(clearTime) - log.Info("token expiration monitor started", - "name", cliMgr.name, - "expiredT", expiredT, - "clearTime", clearTime, - "waitDuration", waitDuration) - - timer := time.NewTimer(waitDuration) - select { - case <-ctx.Done(): - timer.Stop() - log.Info("token expiration monitor cancelled", "name", cliMgr.name) - case <-timer.C: - log.Info("clearing cached client before token expiration", - "name", cliMgr.name, - "expiredT", expiredT) - cliMgr.clearCachedCli(loginCli) - } - }() + // go func() { + // now := time.Now() + // clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration + + // // If clear time is too soon (less than 10s from now), set it to 10s from now + // if clearTime.Before(now.Add(10 * time.Second)) { + // clearTime = now.Add(10 * time.Second) + // log.Error("token expiration time is too close, delaying clear time", + // "name", cliMgr.name, + // "expiredT", expiredT, + // "adjustedClearTime", clearTime) + // } + + // waitDuration := time.Until(clearTime) + // log.Info("token expiration monitor started", + // "name", cliMgr.name, + // "expiredT", expiredT, + // "clearTime", clearTime, + // "waitDuration", waitDuration) + + // timer := time.NewTimer(waitDuration) + // select { + // case <-ctx.Done(): + // timer.Stop() + // log.Info("token expiration monitor cancelled", "name", cliMgr.name) + // case <-timer.C: + // log.Info("clearing cached client before token expiration", + // "name", cliMgr.name, + // "expiredT", expiredT) + // cliMgr.clearCachedCli(loginCli) + // } + // }() cliMgr.cachedCli.Unlock() @@ -186,24 +188,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { } } -func (cliMgr *ClientManager) clearCachedCli(cli *upClient) { - cliMgr.cachedCli.Lock() - if cliMgr.cachedCli.cli == cli { - cliMgr.cachedCli.cli = nil - cliMgr.cachedCli.completionCtx = nil - log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) - } - cliMgr.cachedCli.Unlock() -} - -func (cliMgr *ClientManager) OnResp(cli *upClient, resp *http.Response) { - if resp.StatusCode == http.StatusForbidden { - log.Info("cached client cleared due to forbidden response", "name", cliMgr.name) - cliMgr.clearCachedCli(cli) - } -} - -func (cliMgr *ClientManager) GenLoginParam(challenge string) (*types.LoginParameter, error) { +func (cliMgr *ClientManager) genLoginParam(challenge string) (*types.LoginParameter, error) { // Generate public key string publicKeyHex := common.Bytes2Hex(crypto.CompressPubkey(&cliMgr.privKey.PublicKey)) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go new file mode 100644 index 0000000000..0463f3409c --- /dev/null +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -0,0 +1,140 @@ +package proxy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + + ctypes "scroll-tech/common/types" + "scroll-tech/coordinator/internal/types" +) + +// Client wraps an http client with a preset host for coordinator API calls +type proverSession struct { + sync.RWMutex + proverToken string +} + +func (c *proverSession) doProverLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter) (*types.LoginSchema, error) { + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") + } + + // like SDK, we would try one more time if the upstream token is expired + resp, err := cli.ProxyLogin(ctx, param) + if err != nil { + return nil, fmt.Errorf("proxylogin fail: %v", err) + } + + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + cliMgr.Reset(cli) + cli = cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail (secondary try)") + } + + // like SDK, we would try one more time if the upstream token is expired + resp, err = cli.ProxyLogin(ctx, param) + if err != nil { + return nil, fmt.Errorf("proxylogin fail: %v", err) + } + } + + if resp.ErrCode != 0 { + return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + } + + var loginResult types.LoginSchema + if err := resp.DecodeData(&loginResult); err != nil { + return nil, err + } + + return &loginResult, nil +} + +// ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) (*types.LoginSchema, error) { + url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create proxy login request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.loginToken) + + proxyLoginResp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform proxy login request: %w", err) + } + defer proxyLoginResp.Body.Close() + + // Call helper's OnResp method with the response + c.helper.OnResp(c, proxyLoginResp) + + // Parse proxy login response as LoginSchema + if proxyLoginResp.StatusCode == http.StatusOK { + var loginResult types.LoginSchema + if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { + return &loginResult, nil + } + // If parsing fails, still return success but with nil result + return nil, nil + } + + return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) +} + +// GetTask makes a POST request to /v1/get_task with GetTaskParameter +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create get task request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} + +// SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { + url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) + + jsonData, err := json.Marshal(param) + if err != nil { + return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create submit proof request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + return c.httpClient.Do(req) +} From 9df6429d986a2bda98253fd149cc0e6dd5662dd5 Mon Sep 17 00:00:00 2001 From: Ho Date: Sat, 6 Sep 2025 21:50:55 +0900 Subject: [PATCH 18/26] wip --- .../controller/proxy/prover_session.go | 166 ++++++++++-------- 1 file changed, 96 insertions(+), 70 deletions(-) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 0463f3409c..dbd854508b 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -1,11 +1,8 @@ package proxy import ( - "bytes" "context" - "encoding/json" "fmt" - "net/http" "sync" ctypes "scroll-tech/common/types" @@ -15,126 +12,155 @@ import ( // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { sync.RWMutex - proverToken string + proverToken string + phase uint + completionCtx context.Context } -func (c *proverSession) doProverLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter) (*types.LoginSchema, error) { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter, phase uint) error { + c.Lock() + curPhase := c.phase + if c.completionCtx != nil { + waitctx := c.completionCtx + c.Unlock() + select { + case <-waitctx.Done(): + return c.maintainLogin(ctx, cliMgr, param, phase) + case <-ctx.Done(): + return fmt.Errorf("ctx fail") + } + } + + if phase < curPhase { + // outdate login phase, give up + c.Unlock() + return nil + } + + // occupy the update slot + completeCtx, cf := context.WithCancel(ctx) + defer cf() + c.completionCtx = completeCtx + c.Unlock() + cli := cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail") + return fmt.Errorf("get upstream cli fail") } - // like SDK, we would try one more time if the upstream token is expired resp, err := cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + return fmt.Errorf("proxylogin fail: %v", err) } if resp.ErrCode == ctypes.ErrJWTTokenExpired { cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { - return nil, fmt.Errorf("get upstream cli fail (secondary try)") + return fmt.Errorf("get upstream cli fail (secondary try)") } // like SDK, we would try one more time if the upstream token is expired resp, err = cli.ProxyLogin(ctx, param) if err != nil { - return nil, fmt.Errorf("proxylogin fail: %v", err) + return fmt.Errorf("proxylogin fail: %v", err) } } if resp.ErrCode != 0 { - return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + return fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } var loginResult types.LoginSchema if err := resp.DecodeData(&loginResult); err != nil { - return nil, err + return err } - return &loginResult, nil + c.Lock() + defer c.Unlock() + c.proverToken = loginResult.Token + c.completionCtx = nil + + return nil } // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) (*types.LoginSchema, error) { - url := fmt.Sprintf("%s/coordinator/v1/proxy_login", c.baseURL) - - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal proxy login parameter: %w", err) - } +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { + c.RLock() + phase := c.phase + 1 + c.RUnlock() - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, fmt.Errorf("failed to create proxy login request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+c.loginToken) - - proxyLoginResp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to perform proxy login request: %w", err) - } - defer proxyLoginResp.Body.Close() - - // Call helper's OnResp method with the response - c.helper.OnResp(c, proxyLoginResp) - - // Parse proxy login response as LoginSchema - if proxyLoginResp.StatusCode == http.StatusOK { - var loginResult types.LoginSchema - if err := json.NewDecoder(proxyLoginResp.Body).Decode(&loginResult); err == nil { - return &loginResult, nil - } - // If parsing fails, still return success but with nil result - return nil, nil - } - - return nil, fmt.Errorf("proxy login request failed with status: %d", proxyLoginResp.StatusCode) + return c.maintainLogin(ctx, cli, param, phase) } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/coordinator/v1/get_task", c.baseURL) +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { + c.RLock() + phase := c.phase + token := c.proverToken + c.RUnlock() - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal get task parameter: %w", err) + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") } - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + resp, err := cli.GetTask(ctx, param, token) if err != nil { - return nil, fmt.Errorf("failed to create get task request: %w", err) + return nil, err } - req.Header.Set("Content-Type", "application/json") - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) + } + + // like SDK, we would try one more time if the upstream token is expired + return cli.GetTask(ctx, param, token) } - return c.httpClient.Do(req) + return resp, nil } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, token string) (*http.Response, error) { - url := fmt.Sprintf("%s/coordinator/v1/submit_proof", c.baseURL) +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client) (*ctypes.Response, error) { + c.RLock() + phase := c.phase + token := c.proverToken + c.RUnlock() - jsonData, err := json.Marshal(param) - if err != nil { - return nil, fmt.Errorf("failed to marshal submit proof parameter: %w", err) + cli := cliMgr.Client(ctx) + if cli == nil { + return nil, fmt.Errorf("get upstream cli fail") } - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData)) + resp, err := cli.SubmitProof(ctx, param, token) if err != nil { - return nil, fmt.Errorf("failed to create submit proof request: %w", err) + return nil, err } - req.Header.Set("Content-Type", "application/json") - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) + if resp.ErrCode == ctypes.ErrJWTTokenExpired { + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) + } + + // like SDK, we would try one more time if the upstream token is expired + return cli.SubmitProof(ctx, param, token) } - return c.httpClient.Do(req) + return resp, nil } From 78dbe6cde14f6069e92b6e01fa3fcf5166d15ef0 Mon Sep 17 00:00:00 2001 From: Ho Date: Sun, 7 Sep 2025 22:39:32 +0900 Subject: [PATCH 19/26] controller WIP --- coordinator/internal/controller/proxy/auth.go | 243 ++---------------- .../internal/controller/proxy/controller.go | 11 +- .../internal/controller/proxy/get_task.go | 111 ++------ .../controller/proxy/prover_session.go | 183 ++++++++----- .../internal/controller/proxy/submit_proof.go | 58 ++++- 5 files changed, 216 insertions(+), 390 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 2fc1d76aa0..627be4955d 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -2,9 +2,7 @@ package proxy import ( "fmt" - "sync" - "context" "time" jwt "github.com/appleboy/gin-jwt/v2" @@ -20,192 +18,30 @@ import ( // AuthController is login API type AuthController struct { - apiLogin *api.AuthController - clients Clients - userTokenCache *UserTokenCache -} - -type TokenUpdate struct { - PublicKey string - Upstream string - Phase uint - LoginParam types.LoginParameter - CompleteNotify chan<- *types.LoginSchema -} - -type UpstreamTokens struct { - LoginData map[string]*types.LoginSchema - LoginPhase uint - NextLoginPhase uint -} - -type UserTokenCache struct { - sync.RWMutex - data map[string]UpstreamTokens - tokenCacheUpdate chan<- *TokenUpdate -} - -func newUserTokens() UpstreamTokens { - return UpstreamTokens{ - LoginData: make(map[string]*types.LoginSchema), - } -} - -func newUserCache(tokenCacheUpdate chan<- *TokenUpdate) *UserTokenCache { - return &UserTokenCache{ - data: make(map[string]UpstreamTokens), - tokenCacheUpdate: tokenCacheUpdate, - } -} - -// get retrieves UpstreamTokens for a given user key, returns empty if still not exists -func (c *UserTokenCache) Get(userKey string) *UpstreamTokens { - c.RLock() - defer c.RUnlock() - - tokens, exists := c.data[userKey] - if !exists { - return nil - } - - return &tokens -} - -// prepare for a total update via Login request -func (c *UserTokenCache) updatePrepare(userKey string) UpstreamTokens { - c.Lock() - defer c.Unlock() - - if _, exists := c.data[userKey]; !exists { - log.Info("initializing user token cache", "userKey", userKey) - c.data[userKey] = newUserTokens() - } - updated := c.data[userKey] - updated.NextLoginPhase = updated.LoginPhase + 1 - c.data[userKey] = updated - return updated -} - -// partialSet updates a single entry in upstreamTokens for a given user -func (c *UserTokenCache) partialSet(userKey string, upstreamName string, loginSchema *types.LoginSchema, phase uint) { - c.Lock() - defer c.Unlock() - - // Get existing tokens or create new map - tokens, exists := c.data[userKey] - if exists && tokens.NextLoginPhase == phase { - // Update the specific upstream entry - tokens.LoginData[upstreamName] = loginSchema - } -} - -// LoginParameterWithHardForkName constructs new payload for login -type LoginParameterWithUpstreamTokens struct { - *types.LoginParameter - Tokens UpstreamTokens + apiLogin *api.AuthController + clients Clients + proverMgr *ProverManager } const upstreamConnTimeout = time.Second * 2 -const expireTolerant = 10 * time.Minute const LoginParamCache = "login_param" const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" // NewAuthController returns an LoginController instance -func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier) *AuthController { +func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier, proverMgr *ProverManager) *AuthController { loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) - // Create the token cache update channel - tokenCacheUpdateChan := make(chan *TokenUpdate) - authController := &AuthController{ - apiLogin: api.NewAuthControllerWithLogic(loginLogic), - clients: clients, - userTokenCache: newUserCache(tokenCacheUpdateChan), + apiLogin: api.NewAuthControllerWithLogic(loginLogic), + clients: clients, + proverMgr: proverMgr, } - // Launch token cache manager in a separate goroutine - go authController.toeknCacheManager(tokenCacheUpdateChan) - return authController } -func (a *AuthController) TokenCache() *UserTokenCache { return a.userTokenCache } - -func (a *AuthController) doUpdateRequest(ctx context.Context, req *TokenUpdate) (ret *types.LoginSchema) { - if req.CompleteNotify != nil { - defer func(ctx context.Context) { - select { - case <-ctx.Done(): - case req.CompleteNotify <- ret: - } - - }(ctx) - } - - cli := a.clients[req.Upstream] - if cli := cli.Client(ctx); cli != nil { - var err error - if ret, err = cli.ProxyLogin(ctx, &req.LoginParam); err == nil { - a.userTokenCache.partialSet(req.PublicKey, req.Upstream, ret, req.Phase) - } else { - log.Error("proxy login failed during token cache update", - "userKey", req.PublicKey, - "upstream", req.Upstream, - "phase", req.Phase, - "error", err) - } - } - return - -} - -func (a *AuthController) toeknCacheManager(request <-chan *TokenUpdate) { - - ctx := context.TODO() - var managerStatusLock sync.Mutex - managerStatus := make(map[string]map[string]uint) - - for { - req, ok := <-request - if !ok { - return - } - - // ensure the manager request is not outdated - tokens := a.userTokenCache.Get(req.PublicKey) - if tokens == nil { - // Highly not possible, if raise, the reason is unknown, just log the Error - continue - } - phase := tokens.NextLoginPhase - if req.Phase < phase { - // drop the out-dated request - continue - } - - // ensure only one login request is launched for the same phase - managerStatusLock.Lock() - stat, ok := managerStatus[req.Upstream] - if !ok { - managerStatus[req.Upstream] = make(map[string]uint) - stat = managerStatus[req.Upstream] - } - if phase, running := stat[req.PublicKey]; running && phase >= req.Phase { - managerStatusLock.Unlock() - continue - } else { - stat[req.PublicKey] = req.Phase - } - managerStatusLock.Unlock() - - go a.doUpdateRequest(ctx, req) - - } - -} - // Login extended the Login hander in api controller func (a *AuthController) Login(c *gin.Context) (interface{}, error) { @@ -219,54 +55,24 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { return nil, fmt.Errorf("proxy do not support recursive login") } - tokens := a.userTokenCache.updatePrepare(loginParam.PublicKey) - notifies := make([]chan *types.LoginSchema, len(a.clients)) - - for n := range a.clients { - - // Check if we have a valid cached token that hasn't expired - if knownEntry, existed := tokens.LoginData[n]; existed { - timeRemaining := time.Until(knownEntry.Time) - if timeRemaining > expireTolerant { - // Token is still valid enouth, continue to next client - continue - } - } - - notify := make(chan *types.LoginSchema) - notifies = append(notifies, notify) - request := TokenUpdate{ - PublicKey: loginParam.PublicKey, - Upstream: n, - Phase: tokens.NextLoginPhase, - LoginParam: loginParam.LoginParameter, - CompleteNotify: notify, - } - defer close(notify) - select { - case <-c.Done(): - case a.userTokenCache.tokenCacheUpdate <- &request: - } + session := a.proverMgr.GetOrCreate(loginParam.PublicKey) - } + for n, cli := range a.clients { - // collect all request's compeletions - for _, chn := range notifies { - select { - case <-c.Done(): - case <-chn: + if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { + log.Error("proxy login failed during token cache update", + "userKey", loginParam.PublicKey, + "upstream", n, + "error", err) } } - return LoginParameterWithUpstreamTokens{ - LoginParameter: &loginParam.LoginParameter, - Tokens: tokens, - }, nil + return loginParam.LoginParameter, nil } // PayloadFunc returns jwt.MapClaims with {public key, prover name}. func (a *AuthController) PayloadFunc(data interface{}) jwt.MapClaims { - v, ok := data.(LoginParameterWithUpstreamTokens) + v, ok := data.(types.LoginParameter) if !ok { return jwt.MapClaims{} } @@ -316,23 +122,6 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { } if loginParam.PublicKey != "" { - // ensure tokenCache - a.userTokenCache.RLock() - _, exists := a.userTokenCache.data[loginParam.PublicKey] - if !exists { - a.userTokenCache.RUnlock() - a.userTokenCache.Lock() - if _, exists := a.userTokenCache.data[loginParam.PublicKey]; !exists { - log.Info("creating token cache for user after proxy restart", - "publicKey", loginParam.PublicKey, - "proverName", loginParam.Message.ProverName, - "reason", "prover using JWT token from before proxy restart") - a.userTokenCache.data[loginParam.PublicKey] = newUserTokens() - } - a.userTokenCache.Unlock() - } else { - a.userTokenCache.RUnlock() - } c.Set(LoginParamCache, loginParam) return loginParam.PublicKey diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 5c42927685..122caa85f0 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -1,6 +1,7 @@ package proxy import ( + "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" @@ -21,7 +22,7 @@ var ( type Clients map[string]Client // InitController inits Controller with database -func InitController(cfg *config.ProxyConfig) { +func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { // normalize cfg cfg.ProxyManager.Normalize() @@ -42,7 +43,9 @@ func InitController(cfg *config.ProxyConfig) { clients[nm] = cli } - Auth = NewAuthController(cfg, clients, vf) - // GetTask = NewGetTaskController(cfg, chainCfg, db, vf, reg) - // SubmitProof = NewSubmitProofController(cfg, chainCfg, db, vf, reg) + proverManager := NewProverManager() + + Auth = NewAuthController(cfg, clients, vf, proverManager) + GetTask = NewGetTaskController(cfg, clients, proverManager, reg) + SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 261219aa2f..fad16e3d78 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -1,12 +1,9 @@ package proxy import ( - "encoding/json" "fmt" - "net/http" "github.com/gin-gonic/gin" - "github.com/mitchellh/mapstructure" "github.com/prometheus/client_golang/prometheus" "github.com/scroll-tech/go-ethereum/log" @@ -16,7 +13,7 @@ import ( coordinatorType "scroll-tech/coordinator/internal/types" ) -func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) { +func getSessionData(ctx *gin.Context) string { publicKeyData, publicKeyExist := ctx.Get(coordinatorType.PublicKey) publicKey, castOk := publicKeyData.(string) @@ -25,25 +22,15 @@ func getSessionData(ctx *gin.Context) (string, *coordinatorType.LoginParameter) log.Warn("get_task parameter fail", "error", nerr) types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) - return "", nil + return "" } - loginParamData, publicKeyExist := ctx.Get(LoginParamCache) - loginParam, castOk := loginParamData.(*coordinatorType.LoginParameter) - if !publicKeyExist || !castOk { - nerr := fmt.Errorf("no login param binding: %v", loginParamData) - log.Warn("get_task parameter fail", "error", nerr) - - types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) - return "", nil - } - - return publicKey, loginParam + return publicKey } // GetTaskController the get prover task api controller type GetTaskController struct { - tokenCache *UserTokenCache + proverMgr *ProverManager clients Clients priorityUpstream map[string]string @@ -51,11 +38,11 @@ type GetTaskController struct { } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.Config, clients Clients, tokenCache *UserTokenCache, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ priorityUpstream: make(map[string]string), - tokenCache: tokenCache, + proverMgr: proverMgr, clients: clients, } } @@ -74,88 +61,30 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { return } - publicKey, loginParam := getSessionData(ctx) - if publicKey == "" || loginParam == nil { + publicKey := getSessionData(ctx) + if publicKey == "" { return } - tokens := ptc.tokenCache.Get(publicKey) - - onClientFail := func(upstream string) { - //TODO: log re-connect request in info level - - request := TokenUpdate{ - PublicKey: publicKey, - Upstream: upstream, - Phase: tokens.LoginPhase, - LoginParam: *loginParam, - CompleteNotify: nil, - } - select { - case <-ctx.Done(): - case ptc.tokenCache.tokenCacheUpdate <- &request: - } - - } + session := ptc.proverMgr.Get(publicKey) + // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp priorityUpstream, exist := ptc.priorityUpstream[publicKey] if exist { cli := ptc.clients[priorityUpstream] - loginSchema := tokens.LoginData[priorityUpstream] - if loginSchema == nil { - onClientFail(priorityUpstream) - } else { - ret, triggerUpdate := getTaskFromClient(ctx, cli, &getTaskParameter, loginSchema.Token) - if ret != nil { - - } else if triggerUpdate { - onClientFail(priorityUpstream) - } + resp, err := session.GetTask(ctx, &getTaskParameter, cli, priorityUpstream) + if err != nil { + types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) + return + } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return } - types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } for n, cli := range ptc.clients { - - } -} - -func getTaskFromClient(ctx *gin.Context, cli Client, param *coordinatorType.GetTaskParameter, token string) (*coordinatorType.GetTaskSchema, bool) { - - theCli := cli.PeekClient() - if theCli == nil { - return nil, true + // return the first task we can get + // TODO: use random array for all clients } - - resp, err := theCli.GetTask(ctx, param, token) - if err != nil { - // log the err in error level - return nil, false - } - - // Parse response - if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized { - unAuth := resp.StatusCode == http.StatusUnauthorized - var respWithData types.Response - // Note: Body is consumed after decoding, caller should not read it again - if err := json.NewDecoder(resp.Body).Decode(&respWithData); err == nil { - if unAuth && respWithData.ErrCode == types.ErrJWTTokenExpired { - return nil, true - } - - var getTaskResult coordinatorType.GetTaskSchema - err = mapstructure.Decode(respWithData.Data, &getTaskResult) - if err != nil { - log.Error("parse get task data fail", "respdata", respWithData.Data) - return nil, false - } - return &getTaskResult, false - } else { - log.Error("parse get task response failed", "error", err) - //fmt.Errorf("login parsing response failed: %v", err) - return nil, false - } - } - - return nil, false } diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index dbd854508b..13657c2b60 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -4,37 +4,84 @@ import ( "context" "fmt" "sync" + "time" ctypes "scroll-tech/common/types" "scroll-tech/coordinator/internal/types" ) +type ProverManager struct { + sync.RWMutex + data map[string]*proverSession +} + +func NewProverManager() *ProverManager { + return &ProverManager{ + data: make(map[string]*proverSession), + } +} + +// get retrieves ProverSession for a given user key, returns empty if still not exists +func (m *ProverManager) Get(userKey string) *proverSession { + m.RLock() + defer m.RUnlock() + + return m.data[userKey] +} + +func (m *ProverManager) GetOrCreate(userKey string) *proverSession { + m.Lock() + defer m.Unlock() + + if ret, ok := m.data[userKey]; ok { + return ret + } + + ret := &proverSession{ + proverToken: make(map[string]loginToken), + } + + m.data[userKey] = ret + return ret +} + +func (m *ProverManager) Set(userKey string, session *proverSession) { + m.Lock() + defer m.Unlock() + + m.data[userKey] = session +} + +type loginToken struct { + *types.LoginSchema + phase uint +} + // Client wraps an http client with a preset host for coordinator API calls type proverSession struct { sync.RWMutex - proverToken string - phase uint + proverToken map[string]loginToken completionCtx context.Context } -func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param *types.LoginParameter, phase uint) error { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (*types.LoginSchema, error) { c.Lock() - curPhase := c.phase + curPhase := c.proverToken[up].phase if c.completionCtx != nil { waitctx := c.completionCtx c.Unlock() select { case <-waitctx.Done(): - return c.maintainLogin(ctx, cliMgr, param, phase) + return c.maintainLogin(ctx, cliMgr, up, param, phase) case <-ctx.Done(): - return fmt.Errorf("ctx fail") + return nil, fmt.Errorf("ctx fail") } } if phase < curPhase { // outdate login phase, give up - c.Unlock() - return nil + defer c.Unlock() + return c.proverToken[up].LoginSchema, nil } // occupy the update slot @@ -45,59 +92,75 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, param cli := cliMgr.Client(ctx) if cli == nil { - return fmt.Errorf("get upstream cli fail") + return nil, fmt.Errorf("get upstream cli fail") } resp, err := cli.ProxyLogin(ctx, param) if err != nil { - return fmt.Errorf("proxylogin fail: %v", err) + return nil, fmt.Errorf("proxylogin fail: %v", err) } if resp.ErrCode == ctypes.ErrJWTTokenExpired { cliMgr.Reset(cli) cli = cliMgr.Client(ctx) if cli == nil { - return fmt.Errorf("get upstream cli fail (secondary try)") + return nil, fmt.Errorf("get upstream cli fail (secondary try)") } // like SDK, we would try one more time if the upstream token is expired resp, err = cli.ProxyLogin(ctx, param) if err != nil { - return fmt.Errorf("proxylogin fail: %v", err) + return nil, fmt.Errorf("proxylogin fail: %v", err) } } if resp.ErrCode != 0 { - return fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) + return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } var loginResult types.LoginSchema if err := resp.DecodeData(&loginResult); err != nil { - return err + return nil, err } c.Lock() defer c.Unlock() - c.proverToken = loginResult.Token + + c.proverToken[up] = loginToken{ + LoginSchema: &loginResult, + phase: phase, + } c.completionCtx = nil - return nil + return &loginResult, nil } +const expireTolerant = 10 * time.Minute + // ProxyLogin makes a POST request to /v1/proxy_login with LoginParameter -func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, param *types.LoginParameter) error { +func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { c.RLock() - phase := c.phase + 1 + existedToken := c.proverToken[up].LoginSchema + phase := c.proverToken[up].phase + 1 c.RUnlock() - return c.maintainLogin(ctx, cli, param, phase) + // Check if we have a valid cached token that hasn't expired + if existedToken != nil { + timeRemaining := time.Until(existedToken.Time) + if timeRemaining > expireTolerant { + // Token is still valid enouth, continue to next client + return nil + } + } + + _, err := c.maintainLogin(ctx, cli, up, param, phase) + return err } // GetTask makes a POST request to /v1/get_task with GetTaskParameter -func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client) (*ctypes.Response, error) { +func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParameter, cliMgr Client, up string) (*ctypes.Response, error) { c.RLock() - phase := c.phase - token := c.proverToken + token := c.proverToken[up] c.RUnlock() cli := cliMgr.Client(ctx) @@ -105,35 +168,36 @@ func (c *proverSession) GetTask(ctx context.Context, param *types.GetTaskParamet return nil, fmt.Errorf("get upstream cli fail") } - resp, err := cli.GetTask(ctx, param, token) - if err != nil { - return nil, err - } - - if resp.ErrCode == ctypes.ErrJWTTokenExpired { - // get param from ctx - loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) - if !ok { - return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") - } - - err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if token.LoginSchema != nil { + resp, err := cli.GetTask(ctx, param, token.Token) if err != nil { - return nil, fmt.Errorf("update prover token fail: %V", err) + return nil, err + } + if resp.ErrCode != ctypes.ErrJWTTokenExpired { + return resp, nil } + } - // like SDK, we would try one more time if the upstream token is expired - return cli.GetTask(ctx, param, token) + // like SDK, we would try one more time if the upstream token is expired + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + newToken, err := c.maintainLogin(ctx, cliMgr, up, loginParam, token.phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) } - return resp, nil + return cli.GetTask(ctx, param, newToken.Token) + } // SubmitProof makes a POST request to /v1/submit_proof with SubmitProofParameter -func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client) (*ctypes.Response, error) { +func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProofParameter, cliMgr Client, up string) (*ctypes.Response, error) { c.RLock() - phase := c.phase - token := c.proverToken + token := c.proverToken[up] c.RUnlock() cli := cliMgr.Client(ctx) @@ -141,26 +205,27 @@ func (c *proverSession) SubmitProof(ctx context.Context, param *types.SubmitProo return nil, fmt.Errorf("get upstream cli fail") } - resp, err := cli.SubmitProof(ctx, param, token) - if err != nil { - return nil, err - } - - if resp.ErrCode == ctypes.ErrJWTTokenExpired { - // get param from ctx - loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) - if !ok { - return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") - } - - err = c.maintainLogin(ctx, cliMgr, loginParam, phase) + if token.LoginSchema != nil { + resp, err := cli.SubmitProof(ctx, param, token.Token) if err != nil { - return nil, fmt.Errorf("update prover token fail: %V", err) + return nil, err + } + if resp.ErrCode != ctypes.ErrJWTTokenExpired { + return resp, nil } + } - // like SDK, we would try one more time if the upstream token is expired - return cli.SubmitProof(ctx, param, token) + // like SDK, we would try one more time if the upstream token is expired + // get param from ctx + loginParam, ok := ctx.Value(LoginParamCache).(*types.LoginParameter) + if !ok { + return nil, fmt.Errorf("Unexpected error, no loginparam ctx value") + } + + newToken, err := c.maintainLogin(ctx, cliMgr, up, loginParam, token.phase) + if err != nil { + return nil, fmt.Errorf("update prover token fail: %V", err) } - return resp, nil + return cli.SubmitProof(ctx, param, newToken.Token) } diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index f2afa12945..1841ad1a4a 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -1,29 +1,69 @@ package proxy import ( + "fmt" + "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/params" - "gorm.io/gorm" + "scroll-tech/common/types" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/submitproof" - "scroll-tech/coordinator/internal/logic/verifier" + coordinatorType "scroll-tech/coordinator/internal/types" ) // SubmitProofController the submit proof api controller type SubmitProofController struct { - submitProofReceiverLogic *submitproof.ProofReceiverLogic + proverMgr *ProverManager + clients Clients } // NewSubmitProofController create the submit proof api controller instance -func NewSubmitProofController(cfg *config.Config, chainCfg *params.ChainConfig, db *gorm.DB, vf *verifier.Verifier, reg prometheus.Registerer) *SubmitProofController { +func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *SubmitProofController { return &SubmitProofController{ - submitProofReceiverLogic: submitproof.NewSubmitProofReceiverLogic(cfg.ProverManager, chainCfg, db, vf, reg), + proverMgr: proverMgr, + clients: clients, } } +func upstreamFromTaskName(taskID string) string { + // TODO + return "" +} + // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { - // TODO: implement proxy submit proof logic -} \ No newline at end of file + var submitParameter coordinatorType.SubmitProofParameter + if err := ctx.ShouldBind(&submitParameter); err != nil { + nerr := fmt.Errorf("prover submitProof parameter invalid, err:%w", err) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + publicKey := getSessionData(ctx) + if publicKey == "" { + return + } + + session := spc.proverMgr.Get(publicKey) + upstream := upstreamFromTaskName(submitParameter.TaskID) + cli, existed := spc.clients[upstream] + if !existed { + // TODO: log error + nerr := fmt.Errorf("Invalid upstream name (%s) from taskID %s", upstream, submitParameter.TaskID) + types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) + return + } + + resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) + if err != nil { + types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) + return + } else if resp.ErrCode != 0 { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return + } else { + types.RenderSuccess(ctx, resp.Data) + return + } +} From a04b64df0362e22f7eb6340b1b3670e6ae80fa6e Mon Sep 17 00:00:00 2001 From: Ho Date: Mon, 8 Sep 2025 22:30:51 +0900 Subject: [PATCH 20/26] routes --- .../internal/controller/proxy/get_task.go | 46 +++++++++++++++---- .../internal/controller/proxy/submit_proof.go | 10 +++- .../internal/middleware/challenge_jwt.go | 6 +-- coordinator/internal/middleware/login_jwt.go | 29 ++++++++++++ coordinator/internal/route/route.go | 20 +++++++- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index fad16e3d78..0f0347a2c2 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -68,23 +68,51 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { session := ptc.proverMgr.Get(publicKey) - // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp - priorityUpstream, exist := ptc.priorityUpstream[publicKey] - if exist { - cli := ptc.clients[priorityUpstream] - resp, err := session.GetTask(ctx, &getTaskParameter, cli, priorityUpstream) + getTask := func(upStream string, cli Client) (tryNext bool) { + resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return } else if resp.ErrCode != types.ErrCoordinatorEmptyProofData { - // simply dispatch the error from upstream to prover - types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + + if resp.ErrCode != 0 { + // simply dispatch the error from upstream to prover + types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) + return + } + + var task coordinatorType.GetTaskSchema + if err = resp.DecodeData(&task); err == nil { + task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) + // TODO: log the new id in debug level + types.RenderSuccess(ctx, &task) + } else { + types.RenderFailure(ctx, types.InternalServerError, fmt.Errorf("decode task fail: %v", err)) + } + + return + } + tryNext = true + return + } + + // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp + priorityUpstream, exist := ptc.priorityUpstream[publicKey] + if exist { + cli := ptc.clients[priorityUpstream] + if cli != nil && !getTask(priorityUpstream, cli) { return + } else if cli == nil { + // TODO: log error } } for n, cli := range ptc.clients { - // return the first task we can get - // TODO: use random array for all clients + if !getTask(n, cli) { + return + } } + + // if all get task failed, throw empty proof resp + types.RenderFailure(ctx, types.ErrCoordinatorEmptyProofData, fmt.Errorf("get empty prover task")) } diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 1841ad1a4a..647097785a 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -2,6 +2,7 @@ package proxy import ( "fmt" + "strings" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" @@ -26,10 +27,17 @@ func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMg } func upstreamFromTaskName(taskID string) string { - // TODO + parts, _, found := strings.Cut(taskID, ":") + if found { + return parts + } return "" } +func formUpstreamWithTaskName(upstream string, taskID string) string { + return fmt.Sprintf("%s:%s", upstream, taskID) +} + // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { var submitParameter coordinatorType.SubmitProofParameter diff --git a/coordinator/internal/middleware/challenge_jwt.go b/coordinator/internal/middleware/challenge_jwt.go index 6ee8254b07..99a58cc8db 100644 --- a/coordinator/internal/middleware/challenge_jwt.go +++ b/coordinator/internal/middleware/challenge_jwt.go @@ -14,7 +14,7 @@ import ( ) // ChallengeMiddleware jwt challenge middleware -func ChallengeMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { +func ChallengeMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ Authenticator: func(c *gin.Context) (interface{}, error) { return nil, nil @@ -30,8 +30,8 @@ func ChallengeMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { } }, Unauthorized: unauthorized, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.ChallengeExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.ChallengeExpireDurationSec), TokenLookup: "header: Authorization, query: token, cookie: jwt", TokenHeadName: "Bearer", TimeFunc: time.Now, diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 565aff9daf..7421622d1b 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -9,6 +9,7 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/controller/proxy" "scroll-tech/coordinator/internal/types" ) @@ -46,3 +47,31 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { return jwtMiddleware } + +// ProxyLoginMiddleware jwt auth middleware for proxy login +func ProxyLoginMiddleware(conf *config.ProxyConfig) *jwt.GinJWTMiddleware { + jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ + PayloadFunc: api.Auth.PayloadFunc, + IdentityHandler: api.Auth.IdentityHandler, + IdentityKey: types.PublicKey, + Key: []byte(conf.Auth.Secret), + Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Authenticator: proxy.Auth.Login, + Authorizator: nonIdendityAuthorizator, + Unauthorized: unauthorized, + TokenLookup: "header: Authorization, query: token, cookie: jwt", + TokenHeadName: "Bearer", + TimeFunc: time.Now, + LoginResponse: loginResponse, + }) + + if err != nil { + log.Crit("new jwt middleware panic", "error", err) + } + + if errInit := jwtMiddleware.MiddlewareInit(); errInit != nil { + log.Crit("init jwt middleware panic", "error", errInit) + } + + return jwtMiddleware +} diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 2a0383aa8d..6babf7053a 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -8,6 +8,7 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/api" + "scroll-tech/coordinator/internal/controller/proxy" "scroll-tech/coordinator/internal/middleware" ) @@ -25,7 +26,7 @@ func Route(router *gin.Engine, cfg *config.Config, reg prometheus.Registerer) { func v1(router *gin.RouterGroup, conf *config.Config) { r := router.Group("/v1") - challengeMiddleware := middleware.ChallengeMiddleware(conf) + challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) loginMiddleware := middleware.LoginMiddleware(conf) @@ -39,3 +40,20 @@ func v1(router *gin.RouterGroup, conf *config.Config) { r.POST("/submit_proof", api.SubmitProof.SubmitProof) } } + +func v1_proxy(router *gin.RouterGroup, conf *config.ProxyConfig) { + r := router.Group("/v1") + + challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) + r.GET("/challenge", challengeMiddleware.LoginHandler) + + loginMiddleware := middleware.ProxyLoginMiddleware(conf) + r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) + + // need jwt token api + r.Use(loginMiddleware.MiddlewareFunc()) + { + r.POST("/get_task", proxy.GetTask.GetTasks) + r.POST("/submit_proof", proxy.SubmitProof.SubmitProof) + } +} From 272150365772e0dbc05fbf5b1a0b37e6541091a7 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 9 Sep 2025 20:10:18 +0900 Subject: [PATCH 21/26] refining --- coordinator/internal/config/proxy_config.go | 1 - coordinator/internal/controller/proxy/auth.go | 8 +++- .../internal/controller/proxy/client.go | 40 ++++++++++++++----- .../controller/proxy/client_manager.go | 37 +---------------- .../internal/controller/proxy/controller.go | 11 +---- coordinator/internal/logic/auth/login.go | 2 +- coordinator/internal/middleware/login_jwt.go | 12 +++--- coordinator/internal/route/route.go | 17 ++++++-- 8 files changed, 59 insertions(+), 69 deletions(-) diff --git a/coordinator/internal/config/proxy_config.go b/coordinator/internal/config/proxy_config.go index e2d510a29b..1ea8bd295c 100644 --- a/coordinator/internal/config/proxy_config.go +++ b/coordinator/internal/config/proxy_config.go @@ -45,7 +45,6 @@ type UpStream struct { type ProxyConfig struct { ProxyManager *ProxyManager `json:"proxy_manager"` ProxyName string `json:"proxy_name"` - Auth *Auth `json:"auth"` Coordinators map[string]*UpStream `json:"coondiators"` } diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index 627be4955d..e2d20d58f4 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -29,9 +29,13 @@ const ProverTypesKey = "prover_types" const SignatureKey = "prover_signature" // NewAuthController returns an LoginController instance -func NewAuthController(cfg *config.ProxyConfig, clients Clients, vf *verifier.Verifier, proverMgr *ProverManager) *AuthController { +func NewAuthController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager) *AuthController { - loginLogic := auth.NewLoginLogicWithSimpleDEduplicator(cfg.ProxyManager.Verifier, vf) + // use a dummy Verifier to create login logic (we do not use any information in verifier) + dummyVf := verifier.Verifier{ + OpenVMVkMap: make(map[string]struct{}), + } + loginLogic := auth.NewLoginLogicWithSimpleDeduplicator(cfg.ProxyManager.Verifier, &dummyVf) authController := &AuthController{ apiLogin: api.NewAuthControllerWithLogic(loginLogic), diff --git a/coordinator/internal/controller/proxy/client.go b/coordinator/internal/controller/proxy/client.go index e49caa83f3..e3a6ebb456 100644 --- a/coordinator/internal/controller/proxy/client.go +++ b/coordinator/internal/controller/proxy/client.go @@ -30,6 +30,16 @@ func newUpClient(cfg *config.UpStream) *upClient { } } +func (c *upClient) Token() string { + return c.loginToken +} + +// need a parsable schema defination +type loginSchema struct { + Time string `json:"time"` + Token string `json:"token"` +} + // FullLogin performs the complete login process: get challenge then login func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.LoginParameter, error)) (*types.LoginSchema, error) { // Step 1: Get challenge @@ -44,22 +54,24 @@ func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.Logi if err != nil { return nil, fmt.Errorf("failed to get challenge: %w", err) } - defer challengeResp.Body.Close() - if challengeResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("challenge request failed with status: %d", challengeResp.StatusCode) + parsedResp, err := handleHttpResp(challengeResp) + if err != nil { + return nil, err + } else if parsedResp.ErrCode != 0 { + return nil, fmt.Errorf("challenge failed: %d (%s)", parsedResp.ErrCode, parsedResp.ErrMsg) } - // Step 2: Parse challenge response - var loginSchema types.LoginSchema - if err := json.NewDecoder(challengeResp.Body).Decode(&loginSchema); err != nil { + // Ste p2: Parse challenge response + var challengeSchema loginSchema + if err := parsedResp.DecodeData(&challengeSchema); err != nil { return nil, fmt.Errorf("failed to parse challenge response: %w", err) } // Step 3: Use the token from challenge as Bearer token for login url = fmt.Sprintf("%s/coordinator/v1/login", c.baseURL) - param, err := genLogin(loginSchema.Token) + param, err := genLogin(challengeSchema.Token) if err != nil { return nil, fmt.Errorf("failed to setup login parameter: %w", err) } @@ -75,26 +87,32 @@ func (c *upClient) Login(ctx context.Context, genLogin func(string) (*types.Logi } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+loginSchema.Token) + req.Header.Set("Authorization", "Bearer "+challengeSchema.Token) loginResp, err := c.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("failed to perform login request: %w", err) } - parsedResp, err := handleHttpResp(loginResp) + parsedResp, err = handleHttpResp(loginResp) if err != nil { return nil, err + } else if parsedResp.ErrCode != 0 { + return nil, fmt.Errorf("login failed: %d (%s)", parsedResp.ErrCode, parsedResp.ErrMsg) } - var loginResult types.LoginSchema + var loginResult loginSchema err = parsedResp.DecodeData(&loginResult) if err != nil { return nil, fmt.Errorf("login parsing data fail: %v", err) } c.loginToken = loginResult.Token - return &loginResult, nil + // TODO: we need to parse time if we start making use of it + + return &types.LoginSchema{ + Token: loginResult.Token, + }, nil } func handleHttpResp(resp *http.Response) (*ctypes.Response, error) { diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index 968c66b328..bab6ef96f9 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -126,7 +126,8 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { // Set new completion context and launch login goroutine ctx, completionDone := context.WithCancel(context.TODO()) loginCli := newUpClient(cliMgr.cfg) - cliMgr.cachedCli.completionCtx = context.WithValue(ctx, "cli", loginCli) + completionCtx = context.WithValue(ctx, "cli", loginCli) + cliMgr.cachedCli.completionCtx = completionCtx // Launch keep-login goroutine go func() { @@ -138,40 +139,6 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { cliMgr.cachedCli.cli = loginCli cliMgr.cachedCli.completionCtx = nil - // Launch waiting thread to clear cached client before expiration - // go func() { - // now := time.Now() - // clearTime := expiredT.Add(-10 * time.Second) // 10s before expiration - - // // If clear time is too soon (less than 10s from now), set it to 10s from now - // if clearTime.Before(now.Add(10 * time.Second)) { - // clearTime = now.Add(10 * time.Second) - // log.Error("token expiration time is too close, delaying clear time", - // "name", cliMgr.name, - // "expiredT", expiredT, - // "adjustedClearTime", clearTime) - // } - - // waitDuration := time.Until(clearTime) - // log.Info("token expiration monitor started", - // "name", cliMgr.name, - // "expiredT", expiredT, - // "clearTime", clearTime, - // "waitDuration", waitDuration) - - // timer := time.NewTimer(waitDuration) - // select { - // case <-ctx.Done(): - // timer.Stop() - // log.Info("token expiration monitor cancelled", "name", cliMgr.name) - // case <-timer.C: - // log.Info("clearing cached client before token expiration", - // "name", cliMgr.name, - // "expiredT", expiredT) - // cliMgr.clearCachedCli(loginCli) - // } - // }() - cliMgr.cachedCli.Unlock() }() diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 122caa85f0..35ff02abf8 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -2,10 +2,8 @@ package proxy import ( "github.com/prometheus/client_golang/prometheus" - "github.com/scroll-tech/go-ethereum/log" "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/logic/verifier" ) var ( @@ -26,13 +24,6 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { // normalize cfg cfg.ProxyManager.Normalize() - vf, err := verifier.NewVerifier(cfg.ProxyManager.Verifier) - if err != nil { - panic("proof receiver new verifier failure") - } - - log.Info("verifier created", "openVmVerifier", vf.OpenVMVkMap) - clients := make(map[string]Client) for nm, upCfg := range cfg.Coordinators { @@ -45,7 +36,7 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { proverManager := NewProverManager() - Auth = NewAuthController(cfg, clients, vf, proverManager) + Auth = NewAuthController(cfg, clients, proverManager) GetTask = NewGetTaskController(cfg, clients, proverManager, reg) SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index d64fe56bc9..4207f3b5be 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -40,7 +40,7 @@ func (s *SimpleDeduplicator) InsertChallenge(ctx context.Context, challengeStrin } // NewLoginLogicWithSimpleDEduplicator new a LoginLogic, do not use db to deduplicate challege -func NewLoginLogicWithSimpleDEduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { +func NewLoginLogicWithSimpleDeduplicator(vcfg *config.VerifierConfig, vf *verifier.Verifier) *LoginLogic { return newLoginLogic(&SimpleDeduplicator{}, vcfg, vf) } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 7421622d1b..9aba202c3e 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -21,13 +21,13 @@ func nonIdendityAuthorizator(data interface{}, _ *gin.Context) bool { } // LoginMiddleware jwt auth middleware -func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { +func LoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ PayloadFunc: api.Auth.PayloadFunc, IdentityHandler: api.Auth.IdentityHandler, IdentityKey: types.PublicKey, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), Authenticator: api.Auth.Login, Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, @@ -49,13 +49,13 @@ func LoginMiddleware(conf *config.Config) *jwt.GinJWTMiddleware { } // ProxyLoginMiddleware jwt auth middleware for proxy login -func ProxyLoginMiddleware(conf *config.ProxyConfig) *jwt.GinJWTMiddleware { +func ProxyLoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ PayloadFunc: api.Auth.PayloadFunc, IdentityHandler: api.Auth.IdentityHandler, IdentityKey: types.PublicKey, - Key: []byte(conf.Auth.Secret), - Timeout: time.Second * time.Duration(conf.Auth.LoginExpireDurationSec), + Key: []byte(auth.Secret), + Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), Authenticator: proxy.Auth.Login, Authorizator: nonIdendityAuthorizator, Unauthorized: unauthorized, diff --git a/coordinator/internal/route/route.go b/coordinator/internal/route/route.go index 6babf7053a..5d9a7c65a4 100644 --- a/coordinator/internal/route/route.go +++ b/coordinator/internal/route/route.go @@ -29,7 +29,7 @@ func v1(router *gin.RouterGroup, conf *config.Config) { challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) - loginMiddleware := middleware.LoginMiddleware(conf) + loginMiddleware := middleware.LoginMiddleware(conf.Auth) r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) // need jwt token api @@ -41,13 +41,24 @@ func v1(router *gin.RouterGroup, conf *config.Config) { } } +// Route register route for coordinator +func ProxyRoute(router *gin.Engine, cfg *config.ProxyConfig, reg prometheus.Registerer) { + router.Use(gin.Recovery()) + + observability.Use(router, "coordinator", reg) + + r := router.Group("coordinator") + + v1_proxy(r, cfg) +} + func v1_proxy(router *gin.RouterGroup, conf *config.ProxyConfig) { r := router.Group("/v1") - challengeMiddleware := middleware.ChallengeMiddleware(conf.Auth) + challengeMiddleware := middleware.ChallengeMiddleware(conf.ProxyManager.Auth) r.GET("/challenge", challengeMiddleware.LoginHandler) - loginMiddleware := middleware.ProxyLoginMiddleware(conf) + loginMiddleware := middleware.ProxyLoginMiddleware(conf.ProxyManager.Auth) r.POST("/login", challengeMiddleware.MiddlewareFunc(), loginMiddleware.LoginHandler) // need jwt token api From 50f3e1a97c462af6e0d0843b1d36c80c1bcf1174 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 9 Sep 2025 22:21:24 +0900 Subject: [PATCH 22/26] fix issues from test --- coordinator/internal/controller/proxy/auth.go | 2 + .../internal/controller/proxy/get_task.go | 2 + .../controller/proxy/prover_session.go | 10 +- coordinator/internal/logic/auth/login.go | 21 ++-- coordinator/internal/middleware/login_jwt.go | 4 +- coordinator/test/proxy_test.go | 111 +++++++++++++++++- 6 files changed, 131 insertions(+), 19 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index e2d20d58f4..ccc32e0aa0 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -128,8 +128,10 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { if loginParam.PublicKey != "" { c.Set(LoginParamCache, loginParam) + fmt.Println("identify", loginParam) return loginParam.PublicKey } + fmt.Println("identify empty") return nil } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 0f0347a2c2..13eef26d4e 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -54,6 +54,7 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { + fmt.Println("start get task") var getTaskParameter coordinatorType.GetTaskParameter if err := ctx.ShouldBind(&getTaskParameter); err != nil { nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) @@ -70,6 +71,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { getTask := func(upStream string, cli Client) (tryNext bool) { resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) + fmt.Println("upstream get task", resp) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 13657c2b60..30c69b9a63 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -118,7 +118,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return nil, fmt.Errorf("upstream fail: %d (%s)", resp.ErrCode, resp.ErrMsg) } - var loginResult types.LoginSchema + var loginResult loginSchema if err := resp.DecodeData(&loginResult); err != nil { return nil, err } @@ -127,12 +127,14 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str defer c.Unlock() c.proverToken[up] = loginToken{ - LoginSchema: &loginResult, - phase: phase, + LoginSchema: &types.LoginSchema{ + Token: loginResult.Token, + }, + phase: phase, } c.completionCtx = nil - return &loginResult, nil + return c.proverToken[up].LoginSchema, nil } const expireTolerant = 10 * time.Minute diff --git a/coordinator/internal/logic/auth/login.go b/coordinator/internal/logic/auth/login.go index 4207f3b5be..f691eb0e85 100644 --- a/coordinator/internal/logic/auth/login.go +++ b/coordinator/internal/logic/auth/login.go @@ -93,16 +93,19 @@ func (l *LoginLogic) CompatiblityCheck(login *types.LoginParameter) error { vks[vk] = struct{}{} } - for _, vk := range login.Message.VKs { - if _, ok := vks[vk]; !ok { - log.Error("vk inconsistency", "prover vk", vk, "prover name", login.Message.ProverName, - "prover_version", login.Message.ProverVersion, "message", login.Message) - if !version.CheckScrollProverVersion(login.Message.ProverVersion) { - return fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", - version.Version, login.Message.ProverVersion) + // new coordinator / proxy do not check vks while login, code only for backward compatibility + if len(vks) != 0 { + for _, vk := range login.Message.VKs { + if _, ok := vks[vk]; !ok { + log.Error("vk inconsistency", "prover vk", vk, "prover name", login.Message.ProverName, + "prover_version", login.Message.ProverVersion, "message", login.Message) + if !version.CheckScrollProverVersion(login.Message.ProverVersion) { + return fmt.Errorf("incompatible prover version. please upgrade your prover, expect version: %s, actual version: %s", + version.Version, login.Message.ProverVersion) + } + // if the prover reports a same prover version + return errors.New("incompatible vk. please check your params files or config files") } - // if the prover reports a same prover version - return errors.New("incompatible vk. please check your params files or config files") } } diff --git a/coordinator/internal/middleware/login_jwt.go b/coordinator/internal/middleware/login_jwt.go index 9aba202c3e..66d9702ac8 100644 --- a/coordinator/internal/middleware/login_jwt.go +++ b/coordinator/internal/middleware/login_jwt.go @@ -51,8 +51,8 @@ func LoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { // ProxyLoginMiddleware jwt auth middleware for proxy login func ProxyLoginMiddleware(auth *config.Auth) *jwt.GinJWTMiddleware { jwtMiddleware, err := jwt.New(&jwt.GinJWTMiddleware{ - PayloadFunc: api.Auth.PayloadFunc, - IdentityHandler: api.Auth.IdentityHandler, + PayloadFunc: proxy.Auth.PayloadFunc, + IdentityHandler: proxy.Auth.IdentityHandler, IdentityKey: types.PublicKey, Key: []byte(auth.Secret), Timeout: time.Second * time.Duration(auth.LoginExpireDurationSec), diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 670d6548ca..3a5a234632 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -2,21 +2,30 @@ package test import ( "context" + "errors" "fmt" + "net/http" "testing" "time" + "github.com/gin-gonic/gin" + "github.com/scroll-tech/da-codec/encoding" "github.com/stretchr/testify/assert" + "scroll-tech/common/types/message" + "scroll-tech/common/version" + "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/controller/proxy" + "scroll-tech/coordinator/internal/route" ) func testProxyClientCfg() *config.ProxyClient { return &config.ProxyClient{ - Secret: "test-secret-key", - ProxyName: "test-proxy", + Secret: "test-secret-key", + ProxyName: "test-proxy", + ProxyVersion: version.Version, } } @@ -57,13 +66,107 @@ func testProxyClient(t *testing.T) { // Client should not be nil if login succeeds // Note: This might be nil if the coordinator is not properly set up for proxy authentication // but the test validates that the Client method completes without panic - t.Logf("Client toke: %v", client) + assert.NotNil(t, client) + assert.NotEmpty(t, client.Token()) + t.Logf("Client token: %s (%v)", client.Token(), client) +} + +var ( + proxyConf *config.ProxyConfig +) + +func setupProxy(t *testing.T, proxyURL string, coordinatorURL []string) *http.Server { + var err error + assert.NoError(t, err) + + coordinators := make(map[string]*config.UpStream) + for i, n := range coordinatorURL { + coordinators[fmt.Sprintf("coordinator_%d", i)] = testProxyUpStreamCfg(n) + } + + tokenTimeout = 60 + proxyConf = &config.ProxyConfig{ + ProxyName: "test_proxy", + ProxyManager: &config.ProxyManager{ + Verifier: &config.VerifierConfig{ + MinProverVersion: "v4.4.89", + Verifiers: []config.AssetConfig{{ + AssetsPath: "", + ForkName: "euclidV2", + }}, + }, + Client: testProxyClientCfg(), + Auth: &config.Auth{ + Secret: "proxy", + ChallengeExpireDurationSec: tokenTimeout, + LoginExpireDurationSec: tokenTimeout, + }, + }, + Coordinators: coordinators, + } + + router := gin.New() + proxy.InitController(proxyConf, nil) + route.ProxyRoute(router, proxyConf, nil) + srv := &http.Server{ + Addr: proxyURL, + Handler: router, + } + go func() { + runErr := srv.ListenAndServe() + if runErr != nil && !errors.Is(runErr, http.ErrServerClosed) { + assert.NoError(t, runErr) + } + }() + time.Sleep(time.Second * 2) + + return srv +} + +func testProxyHandshake(t *testing.T) { + // Setup proxy http server. + proxyURL := randomURL() + proxyHttpHandler := setupProxy(t, proxyURL, []string{}) + defer func() { + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + }() + + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + assert.True(t, chunkProver.healthCheckSuccess(t)) +} + +func testProxyGetTask(t *testing.T) { + // Setup coordinator and http server. + coordinatorURL := randomURL() + collector, httpHandler := setupCoordinator(t, 3, coordinatorURL) + defer func() { + collector.Stop() + assert.NoError(t, httpHandler.Shutdown(context.Background())) + }() + + proxyURL := randomURL() + proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL}) + defer func() { + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + }() + + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) + assert.NoError(t, err) + dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) + assert.NoError(t, err) + err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) + assert.NoError(t, err) + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + code, _ := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + assert.Empty(t, code) } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - t.Run("TestProxyHandshake", testProxyClient) + t.Run("TestProxyClient", testProxyClient) + t.Run("TestProxyHandshake", testProxyHandshake) + t.Run("TestProxyGetTask", testProxyGetTask) } From 92ca7a6b76e4f285245e27985a7314fc4c50279e Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 13:55:38 +0900 Subject: [PATCH 23/26] improve get_task proxy --- coordinator/internal/controller/proxy/auth.go | 16 ++--- .../internal/controller/proxy/controller.go | 3 +- .../internal/controller/proxy/get_task.go | 66 ++++++++++++++++--- .../controller/proxy/prover_session.go | 16 ++--- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/coordinator/internal/controller/proxy/auth.go b/coordinator/internal/controller/proxy/auth.go index ccc32e0aa0..a312384d77 100644 --- a/coordinator/internal/controller/proxy/auth.go +++ b/coordinator/internal/controller/proxy/auth.go @@ -63,12 +63,14 @@ func (a *AuthController) Login(c *gin.Context) (interface{}, error) { for n, cli := range a.clients { - if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { - log.Error("proxy login failed during token cache update", - "userKey", loginParam.PublicKey, - "upstream", n, - "error", err) - } + go func(n string, cli Client) { + if err := session.ProxyLogin(c, cli, n, &loginParam.LoginParameter); err != nil { + log.Error("proxy login failed during token cache update", + "userKey", loginParam.PublicKey, + "upstream", n, + "error", err) + } + }(n, cli) } return loginParam.LoginParameter, nil @@ -128,10 +130,8 @@ func (a *AuthController) IdentityHandler(c *gin.Context) interface{} { if loginParam.PublicKey != "" { c.Set(LoginParamCache, loginParam) - fmt.Println("identify", loginParam) return loginParam.PublicKey } - fmt.Println("identify empty") return nil } diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index 35ff02abf8..d1b32ecaff 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -35,8 +35,9 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { } proverManager := NewProverManager() + priorityManager := NewPriorityUpstreamManager() Auth = NewAuthController(cfg, clients, proverManager) - GetTask = NewGetTaskController(cfg, clients, proverManager, reg) + GetTask = NewGetTaskController(cfg, clients, proverManager, priorityManager, reg) SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index 13eef26d4e..b941a53c7c 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -2,6 +2,8 @@ package proxy import ( "fmt" + "math/rand" + "sync" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" @@ -28,20 +30,56 @@ func getSessionData(ctx *gin.Context) string { return publicKey } +// PriorityUpstreamManager manages priority upstream mappings with thread safety +type PriorityUpstreamManager struct { + sync.RWMutex + data map[string]string +} + +// NewPriorityUpstreamManager creates a new PriorityUpstreamManager +func NewPriorityUpstreamManager() *PriorityUpstreamManager { + return &PriorityUpstreamManager{ + data: make(map[string]string), + } +} + +// Get retrieves the priority upstream for a given key +func (p *PriorityUpstreamManager) Get(key string) (string, bool) { + p.RLock() + defer p.RUnlock() + value, exists := p.data[key] + return value, exists +} + +// Set sets the priority upstream for a given key +func (p *PriorityUpstreamManager) Set(key, value string) { + p.Lock() + defer p.Unlock() + p.data[key] = value +} + +// Delete removes the priority upstream for a given key +func (p *PriorityUpstreamManager) Delete(key string) { + p.Lock() + defer p.Unlock() + delete(p.data, key) +} + // GetTaskController the get prover task api controller type GetTaskController struct { proverMgr *ProverManager clients Clients - priorityUpstream map[string]string + priorityUpstream *PriorityUpstreamManager + workingRnd *rand.Rand getTaskAccessCounter *prometheus.CounterVec } // NewGetTaskController create a get prover task controller -func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *GetTaskController { +func NewGetTaskController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, priorityMgr *PriorityUpstreamManager, reg prometheus.Registerer) *GetTaskController { // TODO: implement proxy get task controller initialization return &GetTaskController{ - priorityUpstream: make(map[string]string), + priorityUpstream: priorityMgr, proverMgr: proverMgr, clients: clients, } @@ -54,7 +92,7 @@ func (ptc *GetTaskController) incGetTaskAccessCounter(ctx *gin.Context) error { // GetTasks get assigned chunk/batch task func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { - fmt.Println("start get task") + var getTaskParameter coordinatorType.GetTaskParameter if err := ctx.ShouldBind(&getTaskParameter); err != nil { nerr := fmt.Errorf("prover task parameter invalid, err:%w", err) @@ -71,7 +109,6 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { getTask := func(upStream string, cli Client) (tryNext bool) { resp, err := session.GetTask(ctx, &getTaskParameter, cli, upStream) - fmt.Println("upstream get task", resp) if err != nil { types.RenderFailure(ctx, types.ErrCoordinatorGetTaskFailure, err) return @@ -86,6 +123,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { var task coordinatorType.GetTaskSchema if err = resp.DecodeData(&task); err == nil { task.TaskID = formUpstreamWithTaskName(upStream, task.TaskID) + ptc.priorityUpstream.Set(publicKey, upStream) // TODO: log the new id in debug level types.RenderSuccess(ctx, &task) } else { @@ -99,7 +137,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } // if the priority upsteam is set, we try this upstream first until get the task resp or no task resp - priorityUpstream, exist := ptc.priorityUpstream[publicKey] + priorityUpstream, exist := ptc.priorityUpstream.Get(publicKey) if exist { cli := ptc.clients[priorityUpstream] if cli != nil && !getTask(priorityUpstream, cli) { @@ -109,8 +147,20 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { } } - for n, cli := range ptc.clients { - if !getTask(n, cli) { + // Create a slice to hold the keys + keys := make([]string, 0, len(ptc.clients)) + for k := range ptc.clients { + keys = append(keys, k) + } + + // Shuffle the keys using a local RNG (avoid deprecated rand.Seed) + rand.Shuffle(len(keys), func(i, j int) { + keys[i], keys[j] = keys[j], keys[i] + }) + + // Iterate over the shuffled keys + for _, n := range keys { + if !getTask(n, ptc.clients[n]) { return } } diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 30c69b9a63..7e2d6dd7c4 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -3,6 +3,7 @@ package proxy import ( "context" "fmt" + "math" "sync" "time" @@ -130,7 +131,7 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str LoginSchema: &types.LoginSchema{ Token: loginResult.Token, }, - phase: phase, + phase: curPhase + 1, } c.completionCtx = nil @@ -143,19 +144,18 @@ const expireTolerant = 10 * time.Minute func (c *proverSession) ProxyLogin(ctx context.Context, cli Client, up string, param *types.LoginParameter) error { c.RLock() existedToken := c.proverToken[up].LoginSchema - phase := c.proverToken[up].phase + 1 c.RUnlock() // Check if we have a valid cached token that hasn't expired if existedToken != nil { - timeRemaining := time.Until(existedToken.Time) - if timeRemaining > expireTolerant { - // Token is still valid enouth, continue to next client - return nil - } + // TODO: how to reduce the unnecessary re-login? + // timeRemaining := time.Until(existedToken.Time) + // if timeRemaining > expireTolerant { + // return nil + // } } - _, err := c.maintainLogin(ctx, cli, up, param, phase) + _, err := c.maintainLogin(ctx, cli, up, param, math.MaxUint) return err } From c7b83a0784df9f2b811a65a76edfbd5f41cb4fe3 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 13:55:45 +0900 Subject: [PATCH 24/26] fix issue in test --- coordinator/test/api_test.go | 27 +++++++++++++++++++++++++++ coordinator/test/mock_prover.go | 2 +- coordinator/test/proxy_test.go | 18 +++++++++++++----- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/coordinator/test/api_test.go b/coordinator/test/api_test.go index 053f6b715e..f435dd16b6 100644 --- a/coordinator/test/api_test.go +++ b/coordinator/test/api_test.go @@ -51,6 +51,8 @@ var ( chunk *encoding.Chunk batch *encoding.Batch tokenTimeout int + + envSet bool ) func TestMain(m *testing.M) { @@ -67,6 +69,25 @@ func randomURL() string { return fmt.Sprintf("localhost:%d", 10000+2000+id.Int64()) } +// Generate a batch of random localhost URLs with different ports, similar to randomURL. +func randmURLBatch(n int) []string { + if n <= 0 { + return nil + } + urls := make([]string, 0, n) + used := make(map[int64]struct{}, n) + for len(urls) < n { + id, _ := rand.Int(rand.Reader, big.NewInt(2000-1)) + port := 10000 + 2000 + id.Int64() + if _, ok := used[port]; ok { + continue + } + used[port] = struct{}{} + urls = append(urls, fmt.Sprintf("localhost:%d", port)) + } + return urls +} + func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL string) (*cron.Collector, *http.Server) { var err error db, err = testApps.GetGormDBClient() @@ -130,6 +151,11 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri } func setEnv(t *testing.T) { + if envSet { + t.Log("SetEnv is re-entried") + return + } + var err error version.Version = "v4.4.89" @@ -169,6 +195,7 @@ func setEnv(t *testing.T) { assert.NoError(t, err) batch = &encoding.Batch{Chunks: []*encoding.Chunk{chunk}} + envSet = true } func TestApis(t *testing.T) { diff --git a/coordinator/test/mock_prover.go b/coordinator/test/mock_prover.go index 0076199b33..958c230547 100644 --- a/coordinator/test/mock_prover.go +++ b/coordinator/test/mock_prover.go @@ -191,7 +191,7 @@ func (r *mockProver) tryGetProverTask(t *testing.T, proofType message.ProofType) resp, err := client.R(). SetHeader("Content-Type", "application/json"). SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)). - SetBody(map[string]interface{}{"prover_height": 100, "task_type": int(proofType), "universal": true}). + SetBody(map[string]interface{}{"prover_height": 100, "task_types": []int{int(proofType)}, "universal": true}). SetResult(&result). Post("http://" + r.coordinatorURL + "/coordinator/v1/get_task") assert.NoError(t, err) diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 3a5a234632..4318c96a0d 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -137,14 +137,15 @@ func testProxyHandshake(t *testing.T) { func testProxyGetTask(t *testing.T) { // Setup coordinator and http server. - coordinatorURL := randomURL() + urls := randmURLBatch(2) + coordinatorURL := urls[0] collector, httpHandler := setupCoordinator(t, 3, coordinatorURL) defer func() { collector.Stop() assert.NoError(t, httpHandler.Shutdown(context.Background())) }() - proxyURL := randomURL() + proxyURL := urls[1] proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL}) defer func() { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) @@ -157,16 +158,23 @@ func testProxyGetTask(t *testing.T) { err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) assert.NoError(t, err) + time.Sleep(time.Second) + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) - code, _ := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) assert.Empty(t, code) + if code == 0 { + t.Log("get task id", task.TaskID) + } else { + t.Log("get task error msg", msg) + } } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - t.Run("TestProxyClient", testProxyClient) - t.Run("TestProxyHandshake", testProxyHandshake) + //t.Run("TestProxyClient", testProxyClient) + //t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) } From 057e22072c52fee350a387412dc9b62b058efd2c Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 20:38:21 +0900 Subject: [PATCH 25/26] fix issues --- common/types/response.go | 11 ++++- .../internal/controller/proxy/controller.go | 2 +- .../internal/controller/proxy/get_task.go | 1 + .../controller/proxy/prover_session.go | 28 ++++++----- .../internal/controller/proxy/submit_proof.go | 25 ++++++---- coordinator/internal/types/response_test.go | 48 +++++++++++++++++++ 6 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 coordinator/internal/types/response_test.go diff --git a/common/types/response.go b/common/types/response.go index abd29041e7..714c0dcb05 100644 --- a/common/types/response.go +++ b/common/types/response.go @@ -15,7 +15,16 @@ type Response struct { } func (resp *Response) DecodeData(out interface{}) error { - return mapstructure.Decode(resp.Data, out) + // Decode generically unmarshaled JSON (map[string]any, []any) into a typed struct + // honoring `json` tags and allowing weak type conversions. + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + TagName: "json", + Result: out, + }) + if err != nil { + return err + } + return dec.Decode(resp.Data) } // RenderJSON renders response with json diff --git a/coordinator/internal/controller/proxy/controller.go b/coordinator/internal/controller/proxy/controller.go index d1b32ecaff..0e1b217a3f 100644 --- a/coordinator/internal/controller/proxy/controller.go +++ b/coordinator/internal/controller/proxy/controller.go @@ -39,5 +39,5 @@ func InitController(cfg *config.ProxyConfig, reg prometheus.Registerer) { Auth = NewAuthController(cfg, clients, proverManager) GetTask = NewGetTaskController(cfg, clients, proverManager, priorityManager, reg) - SubmitProof = NewSubmitProofController(cfg, clients, proverManager, reg) + SubmitProof = NewSubmitProofController(cfg, clients, proverManager, priorityManager, reg) } diff --git a/coordinator/internal/controller/proxy/get_task.go b/coordinator/internal/controller/proxy/get_task.go index b941a53c7c..8261ff0f16 100644 --- a/coordinator/internal/controller/proxy/get_task.go +++ b/coordinator/internal/controller/proxy/get_task.go @@ -146,6 +146,7 @@ func (ptc *GetTaskController) GetTasks(ctx *gin.Context) { // TODO: log error } } + ptc.priorityUpstream.Delete(publicKey) // Create a slice to hold the keys keys := make([]string, 0, len(ptc.clients)) diff --git a/coordinator/internal/controller/proxy/prover_session.go b/coordinator/internal/controller/proxy/prover_session.go index 7e2d6dd7c4..eb41fbe3d4 100644 --- a/coordinator/internal/controller/proxy/prover_session.go +++ b/coordinator/internal/controller/proxy/prover_session.go @@ -65,7 +65,7 @@ type proverSession struct { completionCtx context.Context } -func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (*types.LoginSchema, error) { +func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up string, param *types.LoginParameter, phase uint) (result *types.LoginSchema, nerr error) { c.Lock() curPhase := c.proverToken[up].phase if c.completionCtx != nil { @@ -89,6 +89,17 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str completeCtx, cf := context.WithCancel(ctx) defer cf() c.completionCtx = completeCtx + defer func() { + c.Lock() + c.completionCtx = nil + if result != nil { + c.proverToken[up] = loginToken{ + LoginSchema: result, + phase: curPhase + 1, + } + } + c.Unlock() + }() c.Unlock() cli := cliMgr.Client(ctx) @@ -124,18 +135,9 @@ func (c *proverSession) maintainLogin(ctx context.Context, cliMgr Client, up str return nil, err } - c.Lock() - defer c.Unlock() - - c.proverToken[up] = loginToken{ - LoginSchema: &types.LoginSchema{ - Token: loginResult.Token, - }, - phase: curPhase + 1, - } - c.completionCtx = nil - - return c.proverToken[up].LoginSchema, nil + return &types.LoginSchema{ + Token: loginResult.Token, + }, nil } const expireTolerant = 10 * time.Minute diff --git a/coordinator/internal/controller/proxy/submit_proof.go b/coordinator/internal/controller/proxy/submit_proof.go index 647097785a..90c582620b 100644 --- a/coordinator/internal/controller/proxy/submit_proof.go +++ b/coordinator/internal/controller/proxy/submit_proof.go @@ -14,24 +14,26 @@ import ( // SubmitProofController the submit proof api controller type SubmitProofController struct { - proverMgr *ProverManager - clients Clients + proverMgr *ProverManager + clients Clients + priorityUpstream *PriorityUpstreamManager } // NewSubmitProofController create the submit proof api controller instance -func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, reg prometheus.Registerer) *SubmitProofController { +func NewSubmitProofController(cfg *config.ProxyConfig, clients Clients, proverMgr *ProverManager, priorityMgr *PriorityUpstreamManager, reg prometheus.Registerer) *SubmitProofController { return &SubmitProofController{ - proverMgr: proverMgr, - clients: clients, + proverMgr: proverMgr, + clients: clients, + priorityUpstream: priorityMgr, } } -func upstreamFromTaskName(taskID string) string { - parts, _, found := strings.Cut(taskID, ":") +func upstreamFromTaskName(taskID string) (string, string) { + parts, rest, found := strings.Cut(taskID, ":") if found { - return parts + return parts, rest } - return "" + return "", parts } func formUpstreamWithTaskName(upstream string, taskID string) string { @@ -40,6 +42,7 @@ func formUpstreamWithTaskName(upstream string, taskID string) string { // SubmitProof prover submit the proof to coordinator func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { + var submitParameter coordinatorType.SubmitProofParameter if err := ctx.ShouldBind(&submitParameter); err != nil { nerr := fmt.Errorf("prover submitProof parameter invalid, err:%w", err) @@ -53,7 +56,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { } session := spc.proverMgr.Get(publicKey) - upstream := upstreamFromTaskName(submitParameter.TaskID) + upstream, realTaskID := upstreamFromTaskName(submitParameter.TaskID) cli, existed := spc.clients[upstream] if !existed { // TODO: log error @@ -61,6 +64,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { types.RenderFailure(ctx, types.ErrCoordinatorParameterInvalidNo, nerr) return } + submitParameter.TaskID = realTaskID resp, err := session.SubmitProof(ctx, &submitParameter, cli, upstream) if err != nil { @@ -71,6 +75,7 @@ func (spc *SubmitProofController) SubmitProof(ctx *gin.Context) { types.RenderFailure(ctx, resp.ErrCode, fmt.Errorf("%s", resp.ErrMsg)) return } else { + spc.priorityUpstream.Delete(upstream) types.RenderSuccess(ctx, resp.Data) return } diff --git a/coordinator/internal/types/response_test.go b/coordinator/internal/types/response_test.go new file mode 100644 index 0000000000..6508d870a6 --- /dev/null +++ b/coordinator/internal/types/response_test.go @@ -0,0 +1,48 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" + + "scroll-tech/common/types" +) + +func TestResponseDecodeData_GetTaskSchema(t *testing.T) { + // Arrange: build a dummy payload and wrap it in Response + in := GetTaskSchema{ + UUID: "uuid-123", + TaskID: "task-abc", + TaskType: 1, + UseSnark: true, + TaskData: "dummy-data", + HardForkName: "cancun", + } + + resp := types.Response{ + ErrCode: 0, + ErrMsg: "", + Data: in, + } + + // Act: JSON round-trip the Response to simulate real HTTP encoding/decoding + b, err := json.Marshal(resp) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + + var decoded types.Response + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + var out GetTaskSchema + if err := decoded.DecodeData(&out); err != nil { + t.Fatalf("DecodeData error: %v", err) + } + + // Assert: structs match after decode + if !reflect.DeepEqual(in, out) { + t.Fatalf("decoded struct mismatch:\nwant: %+v\n got: %+v", in, out) + } +} From b7f23c6734011a9b1316bd4c43d3cb1dda6b0e68 Mon Sep 17 00:00:00 2001 From: Ho Date: Wed, 10 Sep 2025 20:48:21 +0900 Subject: [PATCH 26/26] basic tests --- .../controller/proxy/client_manager.go | 3 +- coordinator/test/proxy_test.go | 101 +++++++++++++++++- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/coordinator/internal/controller/proxy/client_manager.go b/coordinator/internal/controller/proxy/client_manager.go index bab6ef96f9..ad170384f9 100644 --- a/coordinator/internal/controller/proxy/client_manager.go +++ b/coordinator/internal/controller/proxy/client_manager.go @@ -132,8 +132,7 @@ func (cliMgr *ClientManager) Client(ctx context.Context) *upClient { // Launch keep-login goroutine go func() { defer completionDone() - expiredT := cliMgr.doLogin(context.Background(), loginCli) - log.Info("login compeleted", "name", cliMgr.name, "expired", expiredT) + cliMgr.doLogin(context.Background(), loginCli) cliMgr.cachedCli.Lock() cliMgr.cachedCli.cli = loginCli diff --git a/coordinator/test/proxy_test.go b/coordinator/test/proxy_test.go index 4318c96a0d..b8a09afbac 100644 --- a/coordinator/test/proxy_test.go +++ b/coordinator/test/proxy_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/scroll-tech/da-codec/encoding" "github.com/stretchr/testify/assert" + "scroll-tech/common/types" "scroll-tech/common/types/message" "scroll-tech/common/version" @@ -151,6 +153,10 @@ func testProxyGetTask(t *testing.T) { assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) }() + chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) + code, msg := chunkProver.tryGetProverTask(t, message.ProofTypeChunk) + assert.Equal(t, int(types.ErrCoordinatorEmptyProofData), code) + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) assert.NoError(t, err) dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) @@ -158,23 +164,110 @@ func testProxyGetTask(t *testing.T) { err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) assert.NoError(t, err) - time.Sleep(time.Second) + task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) + assert.Empty(t, code) + if code == 0 { + t.Log("get task id", task.TaskID) + } else { + t.Log("get task error msg", msg) + } + +} + +func testProxyProof(t *testing.T) { + urls := randmURLBatch(3) + coordinatorURL0 := urls[0] + collector0, httpHandler0 := setupCoordinator(t, 3, coordinatorURL0) + defer func() { + collector0.Stop() + httpHandler0.Shutdown(context.Background()) + }() + coordinatorURL1 := urls[1] + collector1, httpHandler1 := setupCoordinator(t, 3, coordinatorURL1) + defer func() { + collector1.Stop() + httpHandler1.Shutdown(context.Background()) + }() + coordinators := map[string]*http.Server{ + "coordinator_0": httpHandler0, + "coordinator_1": httpHandler1, + } + + proxyURL := urls[2] + proxyHttpHandler := setupProxy(t, proxyURL, []string{coordinatorURL0, coordinatorURL1}) + defer func() { + fmt.Println("px end start") + assert.NoError(t, proxyHttpHandler.Shutdown(context.Background())) + fmt.Println("px end") + }() + + err := l2BlockOrm.InsertL2Blocks(context.Background(), []*encoding.Block{block1, block2}) + assert.NoError(t, err) + dbChunk, err := chunkOrm.InsertChunk(context.Background(), chunk) + assert.NoError(t, err) + err = l2BlockOrm.UpdateChunkHashInRange(context.Background(), 0, 100, dbChunk.Hash) + assert.NoError(t, err) chunkProver := newMockProver(t, "prover_chunk_test", proxyURL, message.ProofTypeChunk, version.Version) task, code, msg := chunkProver.getProverTask(t, message.ProofTypeChunk) assert.Empty(t, code) if code == 0 { - t.Log("get task id", task.TaskID) + t.Log("get task", task) + parts, _, _ := strings.Cut(task.TaskID, ":") + // close the coordinator which do not dispatch task first, so if we submit to wrong target, + // there would be a chance the submit failed (to the closed coordinator) + for n, srv := range coordinators { + if n != parts { + t.Log("close coordinator", n) + assert.NoError(t, srv.Shutdown(context.Background())) + } + } + exceptProofStatus := verifiedSuccess + chunkProver.submitProof(t, task, exceptProofStatus, types.Success) + } else { t.Log("get task error msg", msg) } + + // verify proof status + var ( + tick = time.Tick(1500 * time.Millisecond) + tickStop = time.Tick(time.Minute) + ) + + var ( + chunkProofStatus types.ProvingStatus + chunkActiveAttempts int16 + chunkMaxAttempts int16 + ) + + for { + select { + case <-tick: + chunkProofStatus, err = chunkOrm.GetProvingStatusByHash(context.Background(), dbChunk.Hash) + assert.NoError(t, err) + if chunkProofStatus == types.ProvingTaskVerified { + return + } + + chunkActiveAttempts, chunkMaxAttempts, err = chunkOrm.GetAttemptsByHash(context.Background(), dbChunk.Hash) + assert.NoError(t, err) + assert.Equal(t, 1, int(chunkMaxAttempts)) + assert.Equal(t, 0, int(chunkActiveAttempts)) + + case <-tickStop: + t.Error("failed to check proof status", "chunkProofStatus", chunkProofStatus.String()) + return + } + } } func TestProxyClient(t *testing.T) { // Set up the test environment. setEnv(t) - //t.Run("TestProxyClient", testProxyClient) - //t.Run("TestProxyHandshake", testProxyHandshake) + t.Run("TestProxyClient", testProxyClient) + t.Run("TestProxyHandshake", testProxyHandshake) t.Run("TestProxyGetTask", testProxyGetTask) + t.Run("TestProxyValidProof", testProxyProof) }