diff --git a/package.json b/package.json index f0d89874..9947064b 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "biggystring": "^4.1.3", "body-parser": "^1.19.0", "cleaner-config": "^0.1.10", - "cleaners": "^0.3.13", + "cleaners": "^0.3.17", "commander": "^6.1.0", "cors": "^2.8.5", "csv-stringify": "^6.2.0", diff --git a/src/config.ts b/src/config.ts index 9bfc4083..92b904f4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,8 +8,12 @@ export const asConfig = asObject({ ), httpPort: asOptional(asNumber, 8008), bog: asOptional(asObject({ apiKey: asString }), { apiKey: '' }), + + /** Only run specific appIds (e.g. edge, coinhub, etc) */ soloAppIds: asOptional(asArray(asString), null), + /** Only run specific partnerIds (e.g. moonpay, paybis, etc) */ soloPartnerIds: asOptional(asArray(asString), null), + timeoutOverrideMins: asOptional(asNumber, 1200), cacheLookbackMonths: asOptional(asNumber, 24) }) diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts new file mode 100644 index 00000000..434f08fd --- /dev/null +++ b/src/partners/0xgasless.ts @@ -0,0 +1,245 @@ +import { + asArray, + asEither, + asJSON, + asNull, + asNumber, + asObject, + asString, + asUnknown, + asValue +} from 'cleaners' +import URL from 'url-parse' + +import { + asStandardPluginParams, + EDGE_APP_START_DATE, + PartnerPlugin, + PluginParams, + PluginResult, + StandardTx, + Status +} from '../types' +import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' + +const API_URL = 'https://api.0x.org/trade-analytics/gasless' +const PLUGIN_START_DATE = '2024-05-05T00:00:00.000Z' +/** Max fetch retries before bailing */ +const MAX_RETRIES = 5 +/** + * How far to rollback from the last successful query + * date when starting a new query + */ +const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 30 // 30 Days +/** Time period to query per loop */ +const QUERY_TIME_BLOCK_MS = QUERY_LOOKBACK + +export async function query0xGasless( + pluginParams: PluginParams +): Promise { + const { settings, apiKeys } = asStandardPluginParams(pluginParams) + + if (apiKeys.apiKey == null) { + throw new Error('0xGasless: Missing 0xgasless API key') + } + const nowDate = new Date() + const now = nowDate.getTime() + + let { latestIsoDate } = settings + + if (latestIsoDate === EDGE_APP_START_DATE) { + latestIsoDate = new Date(PLUGIN_START_DATE).toISOString() + } + + let lastCheckedTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK + if (lastCheckedTimestamp < 0) lastCheckedTimestamp = 0 + + const ssFormatTxs: StandardTx[] = [] + let retry = 0 + + while (true) { + let latestBlockIsoDate = latestIsoDate + const startTimestamp = lastCheckedTimestamp + const endTimestamp = lastCheckedTimestamp + QUERY_TIME_BLOCK_MS + + try { + let cursor: string | undefined + + while (true) { + const urlObj = new URL(API_URL, true) + + const queryParams: { + startTimestamp: string + endTimestamp: string + cursor?: string + } = { + // API expects seconds-based unix timestamps + startTimestamp: Math.floor(startTimestamp / 1000).toString(), + endTimestamp: Math.floor(endTimestamp / 1000).toString() + } + if (cursor != null) queryParams.cursor = cursor + urlObj.set('query', queryParams) + + datelog( + `0xGasless Querying from:${new Date( + startTimestamp + ).toISOString()} to:${new Date(endTimestamp).toISOString()}` + ) + + const url = urlObj.href + const response = await retryFetch(url, { + headers: { + '0x-api-key': apiKeys.apiKey, + '0x-version': 'v2' + } + }) + const responseJson = await response.text() + if (!response.ok) { + throw new Error(`${url} response ${response.status}: ${responseJson}`) + } + const responseBody = asGetGaslessTradesResponse(responseJson) + + for (const rawTx of responseBody.trades) { + const standardTx = process0xGaslessTx(rawTx) + + ssFormatTxs.push(standardTx) + if (standardTx.isoDate > latestBlockIsoDate) { + latestBlockIsoDate = standardTx.isoDate + } + } + + datelog(`0xGasless ${responseBody.trades.length} trades processed`) + + if (responseBody.nextCursor == null) { + datelog(`0xGasless No cursor from API`) + break + } else { + cursor = responseBody.nextCursor + datelog(`0xGasless Get nextCursor: ${cursor}`) + } + } + + lastCheckedTimestamp = endTimestamp + latestIsoDate = latestBlockIsoDate + datelog( + `0xGasless endDate:${new Date( + lastCheckedTimestamp + ).toISOString()} latestIsoDate:${latestIsoDate}` + ) + if (lastCheckedTimestamp > now) { + break + } + retry = 0 + } catch (error) { + datelog(error) + // Retry a few times with time delay to prevent throttling + retry++ + if (retry <= MAX_RETRIES) { + datelog(`Snoozing ${60 * retry}s`) + await snooze(60000 * retry) + } else { + // We can safely save our progress since we go from oldest to newest, + // and we don't update the lastIsoDate until the page is processed + // fully. + break + } + } + + // Wait before next query, to prevent rate-limiting and thrashing + await snooze(1000) + } + + const out = { + settings: { latestIsoDate }, + transactions: ssFormatTxs + } + return out +} + +export const zeroxgasless: PartnerPlugin = { + queryFunc: query0xGasless, + pluginName: '0xGasless', + pluginId: '0xgasless' +} + +export function process0xGaslessTx(rawTx: unknown): StandardTx { + const trade = asGaslessTrade(rawTx) + const buySymbol = trade.tokens.find(t => t.address === trade.buyToken)?.symbol + const sellSymbol = trade.tokens.find(t => t.address === trade.sellToken) + ?.symbol + + if (buySymbol == null || sellSymbol == null) { + throw new Error( + `Could not find buy or sell symbol for trade with txid ${trade.transactionHash}` + ) + } + + const { + isoDate: tradeIsoDate, + timestamp: tradeTimestamp + } = smartIsoDateFromTimestamp(trade.timestamp * 1000) + + // If trade is 2 days or older, then it's finalized according to 0x + // documentation. + const status: Status = + tradeTimestamp + 2 * 24 * 60 * 60 * 1000 < Date.now() + ? 'complete' + : 'pending' + + const standardTx: StandardTx = { + status, + orderId: trade.transactionHash, + depositTxid: trade.transactionHash, + depositAddress: undefined, + depositCurrency: sellSymbol, + depositAmount: Number(trade.sellAmount), + payoutTxid: trade.transactionHash, + payoutAddress: trade.taker ?? undefined, + payoutCurrency: buySymbol, + payoutAmount: Number(trade.buyAmount), + timestamp: tradeTimestamp, + isoDate: tradeIsoDate, + usdValue: parseFloat(trade.volumeUsd), + rawTx: trade + } + + return standardTx +} + +const asGetGaslessTradesResponse = asJSON( + asObject({ + nextCursor: asEither(asString, asNull), + trades: asArray(asUnknown) + }) +) + +const asGaslessTrade = asObject({ + appName: asString, + blockNumber: asString, + buyToken: asString, + buyAmount: asString, + chainId: asNumber, + // Fee data is not used. + // fees: { + // "integratorFee": null, + // "zeroExFee": null + // }, + gasUsed: asString, + protocolVersion: asString, + sellToken: asString, + sellAmount: asString, + slippageBps: asEither(asString, asNull), + taker: asString, + timestamp: asNumber, + tokens: asArray(v => asGaslessTradeToken(v)), + transactionHash: asString, + volumeUsd: asString, + /** The 0x trade id */ + zid: asString, + service: asValue('gasless') +}) + +const asGaslessTradeToken = asObject({ + address: asString, + symbol: asString +}) diff --git a/src/partners/paybis.ts b/src/partners/paybis.ts index 7e921b53..ae91d3d2 100644 --- a/src/partners/paybis.ts +++ b/src/partners/paybis.ts @@ -175,6 +175,7 @@ export async function queryPaybis( while (true) { const endTime = startTime + QUERY_TIME_BLOCK_MS + let latestBlockIsoDate = latestIsoDate try { let cursor: string | undefined @@ -239,8 +240,8 @@ export async function queryPaybis( rawTx } ssFormatTxs.push(ssTx) - if (ssTx.isoDate > latestIsoDate) { - latestIsoDate = ssTx.isoDate + if (ssTx.isoDate > latestBlockIsoDate) { + latestBlockIsoDate = ssTx.isoDate } } if (cursor == null) { @@ -252,6 +253,7 @@ export async function queryPaybis( const endDate = new Date(endTime) startTime = endTime + latestIsoDate = latestBlockIsoDate datelog( `Paybis endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}` ) diff --git a/src/queryEngine.ts b/src/queryEngine.ts index 1bcc80e6..13b34baf 100644 --- a/src/queryEngine.ts +++ b/src/queryEngine.ts @@ -3,6 +3,7 @@ import nano from 'nano' import { config } from './config' import { pagination } from './dbutils' import { initDbs } from './initDbs' +import { zeroxgasless } from './partners/0xgasless' import { banxa } from './partners/banxa' import { bitaccess } from './partners/bitaccess' import { bitrefill } from './partners/bitrefill' @@ -68,7 +69,8 @@ const plugins = [ thorchain, transak, wyre, - xanpool + xanpool, + zeroxgasless ] const QUERY_FREQ_MS = 60 * 1000 const MAX_CONCURRENT_QUERIES = 3 diff --git a/yarn.lock b/yarn.lock index 3b13a007..4dbe53d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2073,10 +2073,10 @@ cleaner-config@^0.1.10: minimist "^1.2.5" sucrase "^3.17.1" -cleaners@^0.3.13: - version "0.3.13" - resolved "https://registry.npmjs.org/cleaners/-/cleaners-0.3.13.tgz" - integrity sha512-sCedc8LIXUhLmXT9rkkAToi9mjYhI7J/gKRWiF0Qw6eC0ymILHxq+vhuaKoKdcSWpYi2YqqwSlvNtD+92gf4pA== +cleaners@^0.3.17: + version "0.3.17" + resolved "https://registry.yarnpkg.com/cleaners/-/cleaners-0.3.17.tgz#dae498f3d49b7e9364050402d2f4ad09abcd31ba" + integrity sha512-X5acjsLwJK+JEK5hv0Rve7G78+E6iYh1TzJZ40z7Yjrba0WhW6spTq28WgG9w+AK+YQIOHtQTrzaiuntMBBIwQ== cleaners@^0.3.8: version "0.3.16"