feat: add Stork Provider

.gitignore

@@ -38,3 +38,4 @@ run.go
 
 *.log*
 oracle.json
+.aider*

Makefile

@@ -17,33 +17,33 @@ APP_TOML ?= $(HOMEDIR)/config/app.toml
 CONFIG_TOML ?= $(HOMEDIR)/config/config.toml
 COVER_FILE ?= cover.out
 BENCHMARK_ITERS ?= 10
-USE_CORE_MARKETS ?= true
-USE_RAYDIUM_MARKETS ?= false
-USE_UNISWAPV3_BASE_MARKETS ?= false
-USE_COINGECKO_MARKETS ?= false
-USE_COINMARKETCAP_MARKETS ?= false
-USE_OSMOSIS_MARKETS ?= false
-USE_POLYMARKET_MARKETS ?= false
 SCRIPT_DIR := $(CURDIR)/scripts
 DEV_COMPOSE ?= $(CURDIR)/contrib/compose/docker-compose-dev.yml
-
 LEVANT_VAR_FILE:=$(shell mktemp -d)/levant.yaml
 NOMAD_FILE_SLINKY:=contrib/nomad/slinky.nomad
-
 TAG := $(shell git describe --tags --always --dirty)
 
-export HOMEDIR := $(HOMEDIR)
+USE_COINGECKO_MARKETS ?= false
+USE_COINMARKETCAP_MARKETS ?= false
+USE_CORE_MARKETS ?= true
+USE_OSMOSIS_MARKETS ?= false
+USE_POLYMARKET_MARKETS ?= false
+USE_RAYDIUM_MARKETS ?= false
+USE_STORK_MARKETS ?= false
+USE_UNISWAPV3_BASE_MARKETS ?= false
+
 export APP_TOML := $(APP_TOML)
 export GENESIS := $(GENESIS)
 export GENESIS_TMP := $(GENESIS_TMP)
-export USE_CORE_MARKETS ?= $(USE_CORE_MARKETS)
-export USE_RAYDIUM_MARKETS ?= $(USE_RAYDIUM_MARKETS)
-export USE_UNISWAPV3_BASE_MARKETS ?= $(USE_UNISWAPV3_BASE_MARKETS)
+export HOMEDIR := $(HOMEDIR)
+export SCRIPT_DIR := $(SCRIPT_DIR)
 export USE_COINGECKO_MARKETS ?= $(USE_COINGECKO_MARKETS)
 export USE_COINMARKETCAP_MARKETS ?= $(USE_COINMARKETCAP_MARKETS)
+export USE_CORE_MARKETS ?= $(USE_CORE_MARKETS)
 export USE_OSMOSIS_MARKETS ?= $(USE_OSMOSIS_MARKETS)
 export USE_POLYMARKET_MARKETS ?= $(USE_POLYMARKET_MARKETS)
-export SCRIPT_DIR := $(SCRIPT_DIR)
+export USE_RAYDIUM_MARKETS ?= $(USE_RAYDIUM_MARKETS)
+export USE_UNISWAPV3_BASE_MARKETS ?= $(USE_UNISWAPV3_BASE_MARKETS)
 
 BUILD_TAGS := -X github.com/dydxprotocol/slinky/cmd/build.Build=$(TAG)
 

README.md

@@ -101,8 +101,8 @@ We have an extensive suite of metrics available to validators and chain operator
 
 ### Slinky vs. Connect Rename
 
-* Skip renamed their repo (and code refs) from Slinky --> Connect for branding purposes. 
-* Unfortunately it is impossible to update V4 protocol to use the renamed version, as message names etc. were changed. This would cause downtime during the interval when validators have updated `v4-chain` to use the rename but have not yet updated sidecar (or vice-versa). 
+* Skip renamed their repo (and code refs) from Slinky --> Connect for branding purposes.
+* Unfortunately it is impossible to update V4 protocol to use the renamed version, as message names etc. were changed. This would cause downtime during the interval when validators have updated `v4-chain` to use the rename but have not yet updated sidecar (or vice-versa).
 * As a result, the dYdX fork of `skip-mev/connect` has chosen to roll back to the pre-rename state, `slinky`, as the base for future development (along with backporting any essential post-rename changes to functionality).
 
 ### Publishing a Release
@@ -111,10 +111,10 @@ To publish a new release of the `dydxprotocol/slinky` codebase, [follow the offi
 
 ### TODO: Sidecar Deploys
 
-* The GitHub workflows in this repo can build + deploy new images of the Slinky sidecar (along with other images for E2E testing, local testing, etc.) 
+* The GitHub workflows in this repo can build + deploy new images of the Slinky sidecar (along with other images for E2E testing, local testing, etc.)
 * However, these workflows have not yet been updated for dYdX's use case. (Ex. ECR + GHCR URLs still point to Skip's accounts, still rely on Skip secrets, etc.)
-* This is not an urgent issue, unless the sidecar code itself is updated, or breaking changes are made to the format of the Market Map. 
-  * Note: Adding new fields to ex. the `metadata_json` string in Market Map, is not a breaking change (assuming these new fields are not required by the sidecar for price-fetching). 
-* **Next Steps to enable deploys for dYdX's fork of sidecar:** 
-  * Update the URLs and secrets in `build-docker.yml` to deploy to dYdX's ECR / GHCR repos. 
+* This is not an urgent issue, unless the sidecar code itself is updated, or breaking changes are made to the format of the Market Map.
+  * Note: Adding new fields to ex. the `metadata_json` string in Market Map, is not a breaking change (assuming these new fields are not required by the sidecar for price-fetching).
+* **Next Steps to enable deploys for dYdX's fork of sidecar:**
+  * Update the URLs and secrets in `build-docker.yml` to deploy to dYdX's ECR / GHCR repos.
   * Update `v4-chain`'s `docker-compose.yml` to pull from dYdX's repos when building the `slinky0` (sidecar) image.

contrib/compose/docker-compose-dev.yml

@@ -16,6 +16,7 @@ services:
       "--use-coinmarketcap=$USE_COINMARKETCAP_MARKETS",
       "--use-osmosis=$USE_OSMOSIS_MARKETS",
       "--use-polymarket=$USE_POLYMARKET_MARKETS",
+      "--use-stork=$USE_STORK_MARKETS",
       "--temp-file=data/markets.json",
     ]
     environment:
@@ -26,6 +27,7 @@ services:
       - USE_COINMARKETCAP_MARKETS=${USE_COINMARKETCAP_MARKETS:-false}
       - USE_OSMOSIS_MARKETS=${USE_OSMOSIS_MARKETS:-false}
       - USE_POLYMARKET_MARKETS=${USE_POLYMARKET_MARKETS:-false}
+      - USE_STORK_MARKETS=${USE_STORK_MARKETS:-false}
     volumes:
       - markets_data:/data
     networks:
@@ -66,6 +68,7 @@ services:
       - USE_COINGECKO_MARKETS=${USE_COINGECKO_MARKETS:-false}
       - USE_COINMARKETCAP_MARKETS=${USE_COINMARKETCAP_MARKETS:-false}
       - USE_OSMOSIS_MARKETS=${USE_OSMOSIS_MARKETS:-false}
+      - USE_STORK_MARKETS=${USE_STORK_MARKETS:-false}
     build:
       context: ../..
       dockerfile: ./contrib/images/slinky.local.Dockerfile

providers/websockets/stork/README.md

@@ -0,0 +1,91 @@
+# Stork Provider
+
+## Overview
+
+The Stork provider is used to fetch ticker prices from the [Stork Network WebSocket API](https://api.jp.stork-oracle.network/evm/subscribe). Stork provides signed oracle price data with cryptographic proofs for various cryptocurrency assets.
+
+## Authentication
+
+The Stork WebSocket API requires Basic authentication using an API key. The API key should be provided in the configuration or as an environment variable `STORK_API_KEY`.
+
+## Message Types
+
+The provider handles the following message types:
+
+### Subscribe Message
+Used to subscribe to price feeds for specific assets:
+```json
+{
+  "type": "subscribe",
+  "trace_id": "optional_string",
+  "data": ["BTCUSD", "ETHUSD", "BTCUSDMARK"]
+}
+```
+
+### Oracle Prices Message
+Contains the signed price data from Stork:
+```json
+{
+  "type": "oracle_prices",
+  "trace_id": "optional_string",
+  "data": {
+    "BTCUSD": {
+      "stork_signed_price": {
+        "public_key": "...",
+        "encoded_asset_id": "...",
+        "price": "67734000000000000000000",
+        "timestampNs": "1716915868145000000",
+        "evm_signature": "...",
+        "starknet_signature": "..."
+      }
+    }
+  }
+}
+```
+
+## Configuration
+
+Example configuration:
+```json
+{
+  "name": "stork_ws",
+  "enabled": true,
+  "endpoints": [
+    {
+      "url": "wss://api.jp.stork-oracle.network/evm/subscribe",
+      "authentication": {
+        "apiKey": "${STORK_API_KEY}"
+      }
+    }
+  ],
+  "maxBufferSize": 1000,
+  "reconnectionTimeout": "30s",
+  "maxSubscriptionsPerConnection": 100,
+  "maxSubscriptionsPerBatch": 50
+}
+```
+
+## Asset ID Mapping
+
+The provider automatically converts Slinky currency pair formats (e.g., "BTC/USD") to Stork asset ID formats (e.g., "BTCUSD") by removing separators and converting to uppercase.
+
+## Rate Limits
+
+Please refer to Stork's documentation for current rate limits and connection requirements.
+
+## Error Handling
+
+The provider implements comprehensive error handling for:
+- Connection failures and reconnection
+- Message parsing errors
+- Price validation failures
+- Authentication errors
+- Rate limiting
+
+## Supported Features
+
+- Real-time price updates
+- Automatic reconnection
+- Message batching for subscriptions
+- Cryptographic signature verification (signatures are included in responses)
+- Timestamp handling (nanosecond precision)

providers/websockets/stork/messages.go

@@ -0,0 +1,94 @@
+package stork
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"strings"
+
+	slinkymath "github.com/dydxprotocol/slinky/pkg/math"
+	"github.com/dydxprotocol/slinky/providers/base/websocket/handlers"
+)
+
+type (
+	// MessageType represents the type of message sent/received from the websocket.
+	MessageType string
+)
+
+const (
+	// SubscribeMessageType is the message type for subscription requests.
+	SubscribeMessageType MessageType = "subscribe"
+
+	// OraclePricesMessageType is the message type for oracle price updates.
+	OraclePricesMessageType MessageType = "oracle_prices"
+)
+
+// SubscribeMessage represents a subscription request to Stork.
+type SubscribeMessage struct {
+	Type    string   `json:"type"`
+	TraceID string   `json:"trace_id,omitempty"`
+	Data    []string `json:"data"`
+}
+
+// NewSubscribeMessage creates subscription messages for the given assets.
+func (h *WebSocketHandler) NewSubscribeMessage(assets []string) ([]handlers.WebsocketEncodedMessage, error) {
+	numAssets := len(assets)
+	if numAssets == 0 {
+		return nil, fmt.Errorf("no assets to subscribe to")
+	}
+
+	numBatches := int(math.Ceil(float64(numAssets) / float64(h.ws.MaxSubscriptionsPerBatch)))
+	msgs := make([]handlers.WebsocketEncodedMessage, numBatches)
+
+	for i := 0; i < numBatches; i++ {
+		start := i * h.ws.MaxSubscriptionsPerBatch
+		end := slinkymath.Min((i+1)*h.ws.MaxSubscriptionsPerBatch, numAssets)
+
+		msg := SubscribeMessage{
+			Type: string(SubscribeMessageType),
+			Data: assets[start:end],
+		}
+
+		bz, err := json.Marshal(msg)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshal subscribe message: %w", err)
+		}
+		msgs[i] = bz
+	}
+
+	return msgs, nil
+}
+
+// BaseMessage is used to determine the message type.
+type BaseMessage struct {
+	Type string `json:"type"`
+}
+
+// OraclePricesMessage represents the oracle prices response from Stork.
+type OraclePricesMessage struct {
+	Type      string                       `json:"type"`
+	TraceID   string                       `json:"trace_id,omitempty"`
+	Data      map[string]OraclePriceData   `json:"data"`
+}
+
+// OraclePriceData contains the price data for a single asset.
+type OraclePriceData struct {
+	StorkSignedPrice StorkSignedPrice `json:"stork_signed_price"`
+}
+
+// StorkSignedPrice contains the aggregated signed price data.
+type StorkSignedPrice struct {
+	PublicKey         string `json:"public_key"`
+	EncodedAssetID    string `json:"encoded_asset_id"`
+	Price             string `json:"price"`
+	TimestampNS       string `json:"timestampNs"`
+	EVMSignature      string `json:"evm_signature"`
+	StarknetSignature string `json:"starknet_signature"`
+}
+
+// GetAssetID extracts the asset ID from the encoded format.
+func (s *StorkSignedPrice) GetAssetID() string {
+	// The encoded asset ID format needs to be decoded to get the actual asset ID
+	// This is a placeholder - actual implementation depends on Stork's encoding
+	return strings.ToUpper(s.EncodedAssetID)
+}

providers/websockets/stork/parse.go

@@ -0,0 +1,97 @@
+package stork
+
+import (
+	"fmt"
+	"math/big"
+	"strconv"
+	"strings"
+	"time"
+
+	providertypes "github.com/dydxprotocol/slinky/providers/types"
+	"go.uber.org/zap"
+
+	"github.com/dydxprotocol/slinky/oracle/types"
+	"github.com/dydxprotocol/slinky/pkg/math"
+)
+
+// parseOraclePricesMessage parses an oracle prices message from Stork.
+func (h *WebSocketHandler) parseOraclePricesMessage(msg OraclePricesMessage) (types.PriceResponse, error) {
+	var (
+		resolved   = make(types.ResolvedPrices)
+		unResolved = make(types.UnResolvedPrices)
+	)
+
+	for assetID, priceData := range msg.Data {
+		// Get the ticker from the asset ID
+		ticker, ok := h.assetIDToTicker[assetID]
+		if !ok {
+			h.logger.Debug("received price for unknown asset", zap.String("asset_id", assetID))
+			continue
+		}
+
+		// Parse the price
+		price, err := h.parsePrice(priceData.StorkSignedPrice.Price)
+		if err != nil {
+			unResolved[ticker] = providertypes.UnresolvedResult{
+				ErrorWithCode: providertypes.NewErrorWithCode(err, providertypes.ErrorFailedToParsePrice),
+			}
+			continue
+		}
+
+		// Parse the timestamp (convert from nanoseconds)
+		timestamp, err := h.parseTimestamp(priceData.StorkSignedPrice.TimestampNS)
+		if err != nil {
+			unResolved[ticker] = providertypes.UnresolvedResult{
+				ErrorWithCode: providertypes.NewErrorWithCode(err, providertypes.ErrorInvalidResponse),
+			}
+			continue
+		}
+
+		resolved[ticker] = types.NewPriceResult(price, timestamp)
+	}
+
+	return types.NewPriceResponse(resolved, unResolved), nil
+}
+
+// parsePrice converts the Stork price string to a big.Float.
+func (h *WebSocketHandler) parsePrice(priceStr string) (*big.Float, error) {
+	// Stork prices are typically large integers that need to be scaled
+	// The exact scaling factor should be determined from Stork documentation
+	price, err := math.Float64StringToBigFloat(priceStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse price string: %w", err)
+	}
+
+	// Apply scaling if needed (this is a placeholder - actual scaling depends on Stork's format)
+	// For example, if prices are in 18 decimals:
+	// divisor := new(big.Float).SetInt(big.NewInt(1).Exp(big.NewInt(10), big.NewInt(18), nil))
+	// price.Quo(price, divisor)
+
+	return price, nil
+}
+
+// parseTimestamp converts nanosecond timestamp string to time.Time.
+func (h *WebSocketHandler) parseTimestamp(timestampNS string) (time.Time, error) {
+	ns, err := strconv.ParseInt(timestampNS, 10, 64)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to parse timestamp: %w", err)
+	}
+
+	return time.Unix(0, ns).UTC(), nil
+}
+
+// getAssetIDFromTicker converts a provider ticker to Stork asset ID format.
+func (h *WebSocketHandler) getAssetIDFromTicker(ticker types.ProviderTicker) string {
+	// Convert the off-chain ticker format (e.g., "BTC/USD") to Stork format (e.g., "BTCUSD")
+	offChainTicker := ticker.GetOffChainTicker()
+
+	// Remove any separators and convert to uppercase
+	assetID := strings.ReplaceAll(offChainTicker, "/", "")
+	assetID = strings.ReplaceAll(assetID, "-", "")
+	assetID = strings.ToUpper(assetID)
+
+	// Check if there's a custom mapping in the config
+	// This would need to be added to the config structure if needed
+
+	return assetID
+}