From 56f7ed62bff7da921bf0e40b2142d5c784a7c617 Mon Sep 17 00:00:00 2001 From: basarrcan Date: Tue, 21 Apr 2026 16:07:13 +0300 Subject: [PATCH 01/11] feat(api): implement transactions gRPC stream --- core/api-transactions-grpc-stream/BUCK | 20 + core/api-transactions-grpc-stream/Dockerfile | 32 + core/api/BUCK | 8 + core/api/package.json | 6 +- core/api/src/config/env.ts | 20 + core/api/src/config/index.ts | 5 + .../src/domain/transactions-stream/index.ts | 42 ++ .../transactions-stream/index.types.d.ts | 20 + .../transactions-grpc-stream-server.ts | 65 ++ .../services/transactions-stream/convert.ts | 61 ++ .../transactions-stream/grpc-server.ts | 84 +++ .../services/transactions-stream/helpers.ts | 248 +++++++ .../src/services/transactions-stream/index.ts | 208 ++++++ .../transactions-stream/proto/buf.gen.yaml | 14 + .../proto/transactions.proto | 40 + .../proto/transactions_grpc_pb.d.ts | 39 + .../proto/transactions_grpc_pb.js | 44 ++ .../proto/transactions_pb.d.ts | 96 +++ .../proto/transactions_pb.js | 697 ++++++++++++++++++ flake.nix | 13 +- 20 files changed, 1759 insertions(+), 3 deletions(-) create mode 100644 core/api-transactions-grpc-stream/BUCK create mode 100644 core/api-transactions-grpc-stream/Dockerfile create mode 100644 core/api/src/domain/transactions-stream/index.ts create mode 100644 core/api/src/domain/transactions-stream/index.types.d.ts create mode 100644 core/api/src/servers/transactions-grpc-stream-server.ts create mode 100644 core/api/src/services/transactions-stream/convert.ts create mode 100644 core/api/src/services/transactions-stream/grpc-server.ts create mode 100644 core/api/src/services/transactions-stream/helpers.ts create mode 100644 core/api/src/services/transactions-stream/index.ts create mode 100644 core/api/src/services/transactions-stream/proto/buf.gen.yaml create mode 100644 core/api/src/services/transactions-stream/proto/transactions.proto create mode 100644 core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts create mode 100644 core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js create mode 100644 core/api/src/services/transactions-stream/proto/transactions_pb.d.ts create mode 100644 core/api/src/services/transactions-stream/proto/transactions_pb.js diff --git a/core/api-transactions-grpc-stream/BUCK b/core/api-transactions-grpc-stream/BUCK new file mode 100644 index 0000000000..14f0e101ec --- /dev/null +++ b/core/api-transactions-grpc-stream/BUCK @@ -0,0 +1,20 @@ +load( + "@toolchains//workspace-pnpm:macros.bzl", + "prod_tsc_build_bin", +) + +alias( + name = "prod_build", + actual = "//core/api:prod_build", +) + +prod_tsc_build_bin( + name = "api-transactions-grpc-stream", + preload_file = "services/tracing.js", + run_file = "servers/transactions-grpc-stream-server.js", +) + +alias( + name = "dev", + actual = "//core/api:dev-transactions-grpc-stream", +) diff --git a/core/api-transactions-grpc-stream/Dockerfile b/core/api-transactions-grpc-stream/Dockerfile new file mode 100644 index 0000000000..7c991c940d --- /dev/null +++ b/core/api-transactions-grpc-stream/Dockerfile @@ -0,0 +1,32 @@ +FROM nixos/nix:latest AS builder +ARG APP=api-transactions-grpc-stream + +COPY . /workdir +WORKDIR /workdir + +RUN set -eux; \ + nix \ + --extra-experimental-features "nix-command flakes impure-derivations ca-derivations" \ + --option filter-syscalls false \ + build \ + ".#$APP"; + +RUN mkdir -p /tmp/nix-store-closure /tmp/local-bin +RUN cp -R $(nix-store --query --requisites result/) /tmp/nix-store-closure +RUN ln -snf $(nix-store --query result/)/bin/* /tmp/local-bin/ + +FROM gcr.io/distroless/static-debian11 AS final +ARG APP=api-transactions-grpc-stream + +WORKDIR /app/$APP +COPY --from=builder /tmp/nix-store-closure /nix/store +COPY --from=builder /tmp/local-bin/* /usr/local/bin/ + +USER 1000 + +ARG COMMITHASH +ENV COMMITHASH ${COMMITHASH} + +CMD [ \ + "/usr/local/bin/run" \ + ] diff --git a/core/api/BUCK b/core/api/BUCK index fb77ea58eb..c79068a38e 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -287,6 +287,14 @@ dev_pnpm_task_binary( visibility = ["PUBLIC"], ) +dev_pnpm_task_binary( + name = "dev-transactions-grpc-stream", + command = "dev:api-transactions-grpc-stream", + srcs = [":src"], + deps = ["//:node_modules"], + visibility = ["PUBLIC"], +) + dev_pnpm_task_binary( name = "dev-cron", command = "dev:cron", diff --git a/core/api/package.json b/core/api/package.json index 7ee3e92a96..8065e2c3e3 100644 --- a/core/api/package.json +++ b/core/api/package.json @@ -5,7 +5,7 @@ "eslint-check": "eslint src test --ext .ts", "eslint-fix": "eslint src test --ext .ts --fix", "circular-deps-check": "madge --circular --extensions ts src", - "build": "tsc -p tsconfig-build.json && cp -R src/services/price/protos dist/services/price/ && cp -R src/services/dealer-price/proto dist/services/dealer-price/ && cp -R src/services/bria/proto dist/services/bria/ && cp -R src/services/notifications/proto dist/services/notifications/ && cp -R src/services/api-keys/proto dist/services/api-keys/ && tscpaths --silent -p tsconfig.json -s ./src -o ./dist", + "build": "tsc -p tsconfig-build.json && cp -R src/services/price/protos dist/services/price/ && cp -R src/services/dealer-price/proto dist/services/dealer-price/ && cp -R src/services/bria/proto dist/services/bria/ && cp -R src/services/notifications/proto dist/services/notifications/ && cp -R src/services/api-keys/proto dist/services/api-keys/ && cp -R src/services/transactions-stream/proto dist/services/transactions-stream/ && tscpaths --silent -p tsconfig.json -s ./src -o ./dist", "trigger": "pnpm run build && node dist/servers/trigger.js | pino-pretty -c -l", "ws": "pnpm run build && node dist/servers/ws-server.js | pino-pretty -c -l", "watch": "nodemon -V -e ts,graphql -w ./src -x pnpm run start", @@ -26,6 +26,7 @@ "dev:api-trigger": "tsx src/servers/trigger.ts", "dev:api-exporter": "tsx src/servers/exporter.ts", "dev:api-ws-server": "tsx src/servers/ws-server.ts", + "dev:api-transactions-grpc-stream": "tsx src/servers/transactions-grpc-stream-server.ts", "dev:cron": "tsx src/servers/cron.ts", "migrate:create": "migrate-mongo create $MIGRATION_NAME -f src/migrations/migrate-mongo-config.js", "migrate:status": "migrate-mongo status -f src/migrations/migrate-mongo-config.js", @@ -33,7 +34,8 @@ "migrate:down": "migrate-mongo down -f src/migrations/migrate-mongo-config.js", "mongodb-migrate": "pnpm run migrate:status && pnpm run migrate:up && pnpm run migrate:status", "codegen:notifications": "cd ./src/services/notifications/proto && buf generate", - "codegen:api-keys": "cd ./src/services/api-keys/proto && buf generate" + "codegen:api-keys": "cd ./src/services/api-keys/proto && buf generate", + "codegen:transactions-stream": "cd ./src/services/transactions-stream/proto && buf generate" }, "engines": { "node": "20" diff --git a/core/api/src/config/env.ts b/core/api/src/config/env.ts index 881915c9d8..23be8c1be5 100644 --- a/core/api/src/config/env.ts +++ b/core/api/src/config/env.ts @@ -30,6 +30,21 @@ export const env = createEnv({ EXPORTER_PORT: z.number().or(z.string()).pipe(z.coerce.number()).default(3000), TRIGGER_PORT: z.number().or(z.string()).pipe(z.coerce.number()).default(8888), WEBSOCKET_PORT: z.number().or(z.string()).pipe(z.coerce.number()).default(4000), + TRANSACTIONS_GRPC_STREAM_PORT: z + .number() + .or(z.string()) + .pipe(z.coerce.number()) + .default(50053), + TRANSACTIONS_GRPC_STREAM_HEALTH_PORT: z + .number() + .or(z.string()) + .pipe(z.coerce.number()) + .default(8889), + TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS: z + .number() + .or(z.string()) + .pipe(z.coerce.number()) + .default(200), KRATOS_PG_CON: z.string().url(), OATHKEEPER_DECISION_ENDPOINT: z.string().url(), @@ -169,6 +184,11 @@ export const env = createEnv({ EXPORTER_PORT: process.env.EXPORTER_PORT, TRIGGER_PORT: process.env.TRIGGER_PORT, WEBSOCKET_PORT: process.env.WEBSOCKET_PORT, + TRANSACTIONS_GRPC_STREAM_PORT: process.env.TRANSACTIONS_GRPC_STREAM_PORT, + TRANSACTIONS_GRPC_STREAM_HEALTH_PORT: + process.env.TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, + TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS: + process.env.TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS, KRATOS_PG_CON: process.env.KRATOS_PG_CON, OATHKEEPER_DECISION_ENDPOINT: process.env.OATHKEEPER_DECISION_ENDPOINT, diff --git a/core/api/src/config/index.ts b/core/api/src/config/index.ts index 2be58a6437..191d471249 100644 --- a/core/api/src/config/index.ts +++ b/core/api/src/config/index.ts @@ -148,6 +148,11 @@ export const TELEGRAM_PASSPORT_PUBLIC_KEY = getPublicKey(TELEGRAM_PASSPORT_PRIVA export const EXPORTER_PORT = env.EXPORTER_PORT export const TRIGGER_PORT = env.TRIGGER_PORT export const WEBSOCKET_PORT = env.WEBSOCKET_PORT +export const TRANSACTIONS_GRPC_STREAM_PORT = env.TRANSACTIONS_GRPC_STREAM_PORT +export const TRANSACTIONS_GRPC_STREAM_HEALTH_PORT = + env.TRANSACTIONS_GRPC_STREAM_HEALTH_PORT +export const TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS = + env.TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS export const KRATOS_PG_CON = env.KRATOS_PG_CON export const OATHKEEPER_DECISION_ENDPOINT = env.OATHKEEPER_DECISION_ENDPOINT export const GALOY_API_PORT = env.GALOY_API_PORT diff --git a/core/api/src/domain/transactions-stream/index.ts b/core/api/src/domain/transactions-stream/index.ts new file mode 100644 index 0000000000..3c72bc8363 --- /dev/null +++ b/core/api/src/domain/transactions-stream/index.ts @@ -0,0 +1,42 @@ +import { LedgerTransactionType } from "@/domain/ledger" + +export const TransactionsStreamTransactionType = { + Sent: "sent", + Received: "received", +} as const + +export const TransactionsStreamSettlementVia = { + Unspecified: "unspecified", + Lightning: "lightning", + Intraledger: "intraledger", + Onchain: "onchain", +} as const + +export const ledgerTransactionTypeToTransactionsStreamSettlementVia = ( + ledgerTransactionType: LedgerTransactionType, +): TransactionsStreamSettlementVia => { + switch (ledgerTransactionType) { + case LedgerTransactionType.Invoice: + case LedgerTransactionType.Payment: + return TransactionsStreamSettlementVia.Lightning + case LedgerTransactionType.IntraLedger: + case LedgerTransactionType.LnIntraLedger: + case LedgerTransactionType.WalletIdTradeIntraAccount: + case LedgerTransactionType.LnTradeIntraAccount: + return TransactionsStreamSettlementVia.Intraledger + case LedgerTransactionType.OnchainReceipt: + case LedgerTransactionType.OnchainPayment: + case LedgerTransactionType.OnchainIntraLedger: + case LedgerTransactionType.OnChainTradeIntraAccount: + return TransactionsStreamSettlementVia.Onchain + default: + return TransactionsStreamSettlementVia.Unspecified + } +} + +export const ledgerTransactionCreditToTransactionsStreamTransactionType = ( + credit: number, +): TransactionsStreamTransactionType => + credit > 0 + ? TransactionsStreamTransactionType.Received + : TransactionsStreamTransactionType.Sent diff --git a/core/api/src/domain/transactions-stream/index.types.d.ts b/core/api/src/domain/transactions-stream/index.types.d.ts new file mode 100644 index 0000000000..ae8232f2e8 --- /dev/null +++ b/core/api/src/domain/transactions-stream/index.types.d.ts @@ -0,0 +1,20 @@ +type TransactionsStreamTransactionType = + (typeof import("./index").TransactionsStreamTransactionType)[keyof typeof import("./index").TransactionsStreamTransactionType] + +type TransactionsStreamSettlementVia = + (typeof import("./index").TransactionsStreamSettlementVia)[keyof typeof import("./index").TransactionsStreamSettlementVia] + +type TransactionStreamEvent = { + readonly ledgerTransactionId: LedgerTransactionId + readonly walletId: WalletId + readonly accountId: AccountId | undefined + readonly paymentHash: string | undefined + readonly preimage: string | undefined + readonly satsAmount: number + readonly centsAmount: number + readonly currency: WalletCurrency + readonly type: TransactionsStreamTransactionType + readonly settlementVia: TransactionsStreamSettlementVia + readonly pending: boolean + readonly timestamp: Date | undefined +} diff --git a/core/api/src/servers/transactions-grpc-stream-server.ts b/core/api/src/servers/transactions-grpc-stream-server.ts new file mode 100644 index 0000000000..3fa9b34d30 --- /dev/null +++ b/core/api/src/servers/transactions-grpc-stream-server.ts @@ -0,0 +1,65 @@ +import express from "express" + +import { Server, ServerCredentials } from "@grpc/grpc-js" + +import healthzHandler from "./middlewares/healthz" + +import { + TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, + TRANSACTIONS_GRPC_STREAM_PORT, +} from "@/config" + +import { baseLogger } from "@/services/logger" +import { setupMongoConnection } from "@/services/mongodb" +import { TransactionsGrpcServer } from "@/services/transactions-stream/grpc-server" +import { TransactionsStreamService } from "@/services/transactions-stream/proto/transactions_grpc_pb" + +const logger = baseLogger.child({ module: "transactions-grpc-stream-server" }) + +const startHealthServer = () => { + const app = express() + + app.get( + "/healthz", + healthzHandler({ + checkDbConnectionStatus: true, + checkRedisStatus: false, + checkLndsStatus: false, + checkBriaStatus: false, + }), + ) + + app.listen(TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, () => { + logger.info( + { port: TRANSACTIONS_GRPC_STREAM_HEALTH_PORT }, + "Transactions gRPC stream health server listening", + ) + }) +} + +const startGrpcServer = async () => { + const server = new Server() + server.addService(TransactionsStreamService, TransactionsGrpcServer()) + + const address = `0.0.0.0:${TRANSACTIONS_GRPC_STREAM_PORT}` + await new Promise((resolve, reject) => { + server.bindAsync(address, ServerCredentials.createInsecure(), (err) => { + if (err) return reject(err) + return resolve() + }) + }) + + server.start() + logger.info({ address }, "Transactions gRPC stream listening") +} + +const main = async () => { + startHealthServer() + + await setupMongoConnection({ syncIndexes: false }) + await startGrpcServer() +} + +main().catch((err) => { + logger.error({ err }, "Transactions gRPC stream server failed") +}) diff --git a/core/api/src/services/transactions-stream/convert.ts b/core/api/src/services/transactions-stream/convert.ts new file mode 100644 index 0000000000..c9d2c670fe --- /dev/null +++ b/core/api/src/services/transactions-stream/convert.ts @@ -0,0 +1,61 @@ +import { + SettlementViaType, + TransactionEvent, + TransactionType, +} from "./proto/transactions_pb" + +import { + TransactionsStreamSettlementVia, + TransactionsStreamTransactionType, +} from "@/domain/transactions-stream" + +const transactionStreamTransactionTypeToGrpcTransactionType = ( + transactionType: TransactionsStreamTransactionType, +): TransactionType => { + switch (transactionType) { + case TransactionsStreamTransactionType.Sent: + return TransactionType.SENT + case TransactionsStreamTransactionType.Received: + return TransactionType.RECEIVED + } +} + +const transactionStreamSettlementViaToGrpcSettlementVia = ( + settlementVia: TransactionsStreamSettlementVia, +): SettlementViaType => { + switch (settlementVia) { + case TransactionsStreamSettlementVia.Lightning: + return SettlementViaType.LIGHTNING + case TransactionsStreamSettlementVia.Intraledger: + return SettlementViaType.INTRA_LEDGER + case TransactionsStreamSettlementVia.Onchain: + return SettlementViaType.ONCHAIN + case TransactionsStreamSettlementVia.Unspecified: + return SettlementViaType.SETTLEMENT_VIA_UNSPECIFIED + } +} + +export const transactionStreamEventToGrpcTransactionEvent = ( + event: TransactionStreamEvent, +): TransactionEvent => { + const grpcEvent = new TransactionEvent() + + grpcEvent.setLedgerTransactionId(event.ledgerTransactionId) + grpcEvent.setWalletId(event.walletId) + grpcEvent.setAccountId(event.accountId ?? "") + grpcEvent.setPaymentHash(event.paymentHash ?? "") + grpcEvent.setPreimage(event.preimage ?? "") + grpcEvent.setSatsAmount(event.satsAmount) + grpcEvent.setCentsAmount(event.centsAmount) + grpcEvent.setCurrency(event.currency) + grpcEvent.setType(transactionStreamTransactionTypeToGrpcTransactionType(event.type)) + grpcEvent.setSettlementVia( + transactionStreamSettlementViaToGrpcSettlementVia(event.settlementVia), + ) + grpcEvent.setPending(event.pending) + grpcEvent.setTimestamp( + event.timestamp ? Math.floor(event.timestamp.getTime() / 1000) : 0, + ) + + return grpcEvent +} diff --git a/core/api/src/services/transactions-stream/grpc-server.ts b/core/api/src/services/transactions-stream/grpc-server.ts new file mode 100644 index 0000000000..969712164a --- /dev/null +++ b/core/api/src/services/transactions-stream/grpc-server.ts @@ -0,0 +1,84 @@ +import { handleServerStreamingCall, Metadata, ServiceError, status } from "@grpc/grpc-js" + +import { transactionStreamEventToGrpcTransactionEvent } from "./convert" +import { ITransactionsStreamServer } from "./proto/transactions_grpc_pb" +import { SubscribeTransactionsRequest, TransactionEvent } from "./proto/transactions_pb" + +import { + TransactionsStreamService, + TransactionsStreamSubscription, +} from "@/services/transactions-stream" +import { baseLogger } from "@/services/logger" + +const logger = baseLogger.child({ module: "transactions-grpc-stream" }) + +const toServiceError = ({ + code, + message, + details, +}: { + code: status + message: string + details: string +}): ServiceError => + Object.assign(new Error(message), { + code, + details, + metadata: new Metadata(), + }) + +type TransactionsGrpcServerConfig = { + transactionsStreamService?: ReturnType + logger?: Logger +} + +export const TransactionsGrpcServer = ({ + transactionsStreamService = TransactionsStreamService(), + logger: serviceLogger = logger, +}: TransactionsGrpcServerConfig = {}): ITransactionsStreamServer => { + const subscribeTransactions: handleServerStreamingCall< + SubscribeTransactionsRequest, + TransactionEvent + > = (call) => { + let isClosed = false + const subscriptionRef: { current?: TransactionsStreamSubscription } = {} + + const cleanup = () => { + if (isClosed) return + isClosed = true + subscriptionRef.current?.close() + } + + const result = transactionsStreamService.subscribeToTransactions({ + afterTransactionId: call.request.getAfterTransactionId() || undefined, + onTransaction: async (event) => { + if (isClosed) return + call.write(transactionStreamEventToGrpcTransactionEvent(event)) + }, + onError: (err) => { + serviceLogger.error({ err }, "Failed to stream transactions") + cleanup() + call.destroy(err) + }, + }) + + if (result instanceof Error) { + call.destroy( + toServiceError({ + code: status.INVALID_ARGUMENT, + message: "Invalid after_transaction_id", + details: "after_transaction_id must be a valid Mongo ObjectId", + }), + ) + return + } + + subscriptionRef.current = result + + call.on("cancelled", cleanup) + call.on("close", cleanup) + call.on("error", cleanup) + } + + return { subscribeTransactions } +} diff --git a/core/api/src/services/transactions-stream/helpers.ts b/core/api/src/services/transactions-stream/helpers.ts new file mode 100644 index 0000000000..2ac6c0b58d --- /dev/null +++ b/core/api/src/services/transactions-stream/helpers.ts @@ -0,0 +1,248 @@ +import { + ledgerTransactionCreditToTransactionsStreamTransactionType, + ledgerTransactionTypeToTransactionsStreamSettlementVia, +} from "@/domain/transactions-stream" +import { + LedgerTransactionType, + liabilitiesMainAccount, + toWalletId, +} from "@/domain/ledger" + +import { TransactionMetadata } from "@/services/ledger/schema" +import { WalletInvoice } from "@/services/mongoose/schema" +import { WalletsRepository } from "@/services/mongoose/wallets" + +const PREIMAGE_CACHE_TTL_MS = 5 * 60 * 1000 +const PREIMAGE_CACHE_MAX_SIZE = 10_000 + +export const LIABILITIES_ACCOUNT_PREFIX = `${liabilitiesMainAccount}:` +export const LIABILITIES_ACCOUNT_PATTERN = /^Liabilities:/ + +export const EXCLUDED_LEDGER_TRANSACTION_TYPES: LedgerTransactionType[] = [ + LedgerTransactionType.Fee, + LedgerTransactionType.ToColdStorage, + LedgerTransactionType.ToHotWallet, + LedgerTransactionType.Escrow, + LedgerTransactionType.RoutingRevenue, + LedgerTransactionType.Reconciliation, +] + +export const SETTLED_TRANSACTION_FILTER = { + accounts: LIABILITIES_ACCOUNT_PATTERN, + pending: false, + type: { $nin: EXCLUDED_LEDGER_TRANSACTION_TYPES }, +} as const + +type TimestampedValue = { + value: T + expiresAt: number +} + +type PreimageLoaderArgs = { + transactionId: string + paymentHash?: string +} + +export type TransactionStreamRecord = Pick< + ILedgerTransaction, + | "accounts" + | "hash" + | "type" + | "pending" + | "currency" + | "satsAmount" + | "centsAmount" + | "credit" + | "datetime" + | "timestamp" +> & { + _id: ObjectId +} + +export type AccountIdLoader = (walletId: WalletId) => Promise +export type PreimageLoader = (args: PreimageLoaderArgs) => Promise +export type AccountIdResolver = (walletId: WalletId) => Promise +export type PreimageResolver = (args: PreimageLoaderArgs) => Promise + +type FindWalletById = ( + walletId: WalletId, +) => Promise<{ accountId: AccountId } | Error | undefined> +type FindTransactionMetadataById = ( + transactionId: string, +) => Promise | null | undefined> +type FindWalletInvoiceById = ( + paymentHash: string, +) => Promise | null | undefined> + +class ExpiringCache { + private readonly values = new Map>() + + constructor( + private readonly options: { + ttlMs: number + maxSize: number + }, + ) {} + + get(key: K): V | undefined { + const cached = this.values.get(key) + if (!cached) return undefined + + if (cached.expiresAt <= Date.now()) { + this.values.delete(key) + return undefined + } + + return cached.value + } + + set(key: K, value: V) { + if (this.values.size >= this.options.maxSize) { + const oldestKey = this.values.keys().next().value + if (oldestKey !== undefined) this.values.delete(oldestKey) + } + + this.values.set(key, { + value, + expiresAt: Date.now() + this.options.ttlMs, + }) + } +} + +export const parseWalletId = (accounts: string): WalletId | undefined => { + if (!accounts.startsWith(LIABILITIES_ACCOUNT_PREFIX)) return undefined + return toWalletId(accounts as LiabilitiesWalletId) +} + +const defaultFindWalletById: FindWalletById = async (walletId) => + WalletsRepository().findById(walletId) + +const defaultFindTransactionMetadataById: FindTransactionMetadataById = async ( + transactionId, +) => + (await TransactionMetadata.findById(transactionId).lean()) as Pick< + TransactionMetadataRecord, + "revealedPreImage" + > | null + +const defaultFindWalletInvoiceById: FindWalletInvoiceById = async (paymentHash) => + (await WalletInvoice.findById(paymentHash).lean()) as Pick< + WalletInvoiceRecord, + "secret" + > | null + +export const createAccountIdLoader = ({ + findWalletById = defaultFindWalletById, +}: { + findWalletById?: FindWalletById +} = {}): AccountIdLoader => { + return async (walletId) => { + const wallet = await findWalletById(walletId) + if (!wallet || wallet instanceof Error) return undefined + + return wallet.accountId + } +} + +export const createPreimageLoader = ({ + findTransactionMetadataById = defaultFindTransactionMetadataById, + findWalletInvoiceById = defaultFindWalletInvoiceById, +}: { + findTransactionMetadataById?: FindTransactionMetadataById + findWalletInvoiceById?: FindWalletInvoiceById +} = {}): PreimageLoader => { + return async ({ transactionId, paymentHash }) => { + const txMetadata = await findTransactionMetadataById(transactionId) + if (txMetadata?.revealedPreImage) return txMetadata.revealedPreImage + if (!paymentHash) return "" + + const invoice = await findWalletInvoiceById(paymentHash) + return invoice?.secret ?? "" + } +} + +export const createAccountIdResolver = ({ + walletToAccountCache = new Map(), + loadAccountId = createAccountIdLoader(), +}: { + walletToAccountCache?: Map + loadAccountId?: AccountIdLoader +} = {}): AccountIdResolver => { + return async (walletId: WalletId) => { + if (walletToAccountCache.has(walletId)) { + return walletToAccountCache.get(walletId) + } + + const accountId = await loadAccountId(walletId) + walletToAccountCache.set(walletId, accountId) + + return accountId + } +} + +export const createPreimageResolver = ({ + preimageCache = new ExpiringCache({ + ttlMs: PREIMAGE_CACHE_TTL_MS, + maxSize: PREIMAGE_CACHE_MAX_SIZE, + }), + loadPreimage = createPreimageLoader(), +}: { + preimageCache?: { + get: (key: string) => string | undefined + set: (key: string, value: string) => void + } + loadPreimage?: PreimageLoader +} = {}): PreimageResolver => { + return async ({ transactionId, paymentHash }: PreimageLoaderArgs) => { + const cached = preimageCache.get(transactionId) + if (cached !== undefined) return cached + + const preimage = await loadPreimage({ transactionId, paymentHash }) + preimageCache.set(transactionId, preimage) + return preimage + } +} + +export const createTransactionStreamEventMapper = ({ + resolveAccountId = createAccountIdResolver(), + resolvePreimage = createPreimageResolver(), +}: { + resolveAccountId?: AccountIdResolver + resolvePreimage?: PreimageResolver +} = {}) => { + const mapTransactionStreamEvent = async ( + ledgerTransaction: TransactionStreamRecord, + ): Promise => { + const walletId = parseWalletId(ledgerTransaction.accounts) + if (!walletId) return undefined + + const [accountId, preimage] = await Promise.all([ + resolveAccountId(walletId), + resolvePreimage({ + transactionId: ledgerTransaction._id.toString(), + paymentHash: ledgerTransaction.hash, + }), + ]) + + return { + ledgerTransactionId: ledgerTransaction._id.toString() as LedgerTransactionId, + walletId, + accountId, + paymentHash: ledgerTransaction.hash ?? undefined, + preimage, + satsAmount: ledgerTransaction.satsAmount ?? 0, + centsAmount: ledgerTransaction.centsAmount ?? 0, + currency: ledgerTransaction.currency, + type: ledgerTransactionCreditToTransactionsStreamTransactionType( + ledgerTransaction.credit, + ), + settlementVia: ledgerTransactionTypeToTransactionsStreamSettlementVia( + ledgerTransaction.type, + ), + pending: Boolean(ledgerTransaction.pending), + timestamp: ledgerTransaction.datetime ?? ledgerTransaction.timestamp ?? undefined, + } + } + + return { mapTransactionStreamEvent } +} diff --git a/core/api/src/services/transactions-stream/index.ts b/core/api/src/services/transactions-stream/index.ts new file mode 100644 index 0000000000..f4ac1c808d --- /dev/null +++ b/core/api/src/services/transactions-stream/index.ts @@ -0,0 +1,208 @@ +import mongoose from "mongoose" + +import { + createTransactionStreamEventMapper, + SETTLED_TRANSACTION_FILTER, + TransactionStreamRecord, +} from "./helpers" + +import { TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS } from "@/config" +import { checkedToLedgerTransactionId } from "@/domain/ledger" + +import { Transaction } from "@/services/ledger/schema" +import { baseLogger } from "@/services/logger" + +const logger = baseLogger.child({ module: "transactions-stream" }) + +const DEFAULT_BATCH_SIZE = 100 + +const TRANSACTION_STREAM_PROJECTION = { + _id: 1, + accounts: 1, + hash: 1, + type: 1, + pending: 1, + currency: 1, + satsAmount: 1, + centsAmount: 1, + credit: 1, + datetime: 1, + timestamp: 1, +} as const + +const toError = (err: unknown, fallbackMessage: string): Error => { + if (err instanceof Error) return err + return new Error(fallbackMessage) +} + +export type TransactionStreamQueries = { + listSettledTransactionsAfter: (args: { + afterTransactionId: LedgerTransactionId + limit: number + }) => Promise + findLatestTransactionId: () => Promise +} + +type SubscribeToTransactionsArgs = { + afterTransactionId?: string + onTransaction: (event: TransactionStreamEvent) => void | Promise + onError: (err: Error) => void +} + +export type TransactionsStreamSubscription = { + close: () => void +} + +type TransactionsStreamServiceConfig = { + batchSize?: number + pollIntervalMs?: number + transactionQueries?: TransactionStreamQueries + mapTransactionStreamEvent?: ( + ledgerTransaction: TransactionStreamRecord, + ) => Promise + logger?: Logger +} + +export const createTransactionStreamQueries = (): TransactionStreamQueries => { + const listSettledTransactionsAfter = async ({ + afterTransactionId, + limit, + }: { + afterTransactionId: LedgerTransactionId + limit: number + }): Promise => { + return (await Transaction.find( + { + _id: { $gt: new mongoose.Types.ObjectId(afterTransactionId) }, + ...SETTLED_TRANSACTION_FILTER, + }, + TRANSACTION_STREAM_PROJECTION, + ) + .sort({ _id: 1 }) + .limit(limit) + .lean()) as TransactionStreamRecord[] + } + + const findLatestTransactionId = async (): Promise => { + const latest = (await Transaction.findOne({}, { _id: 1 }) + .sort({ _id: -1 }) + .lean()) as { _id?: ObjectId } | null + + return latest?._id?.toString() as LedgerTransactionId | undefined + } + + return { + listSettledTransactionsAfter, + findLatestTransactionId, + } +} + +const parseAfterTransactionId = ( + afterTransactionId?: string, +): LedgerTransactionId | Error | undefined => { + if (!afterTransactionId) return undefined + + const checkedLedgerTransactionId = checkedToLedgerTransactionId(afterTransactionId) + if (checkedLedgerTransactionId instanceof Error) return checkedLedgerTransactionId + + return checkedLedgerTransactionId +} + +export const TransactionsStreamService = ({ + batchSize = DEFAULT_BATCH_SIZE, + pollIntervalMs = TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS, + transactionQueries = createTransactionStreamQueries(), + mapTransactionStreamEvent = createTransactionStreamEventMapper() + .mapTransactionStreamEvent, + logger: serviceLogger = logger, +}: TransactionsStreamServiceConfig = {}) => { + const subscribeToTransactions = ({ + afterTransactionId, + onTransaction, + onError, + }: SubscribeToTransactionsArgs): TransactionsStreamSubscription | Error => { + const parsedCursor = parseAfterTransactionId(afterTransactionId) + if (parsedCursor instanceof Error) return parsedCursor + + let cursor = parsedCursor + let isClosed = false + let pollInFlight = false + let interval: NodeJS.Timeout | undefined + + const cleanup = () => { + if (interval) clearInterval(interval) + interval = undefined + isClosed = true + } + + const streamTransactions = async () => { + if (!cursor) return + + while (!isClosed) { + const ledgerTransactions = await transactionQueries.listSettledTransactionsAfter({ + afterTransactionId: cursor, + limit: batchSize, + }) + + if (ledgerTransactions.length === 0) return + + for (const ledgerTransaction of ledgerTransactions) { + cursor = ledgerTransaction._id.toString() as LedgerTransactionId + const event = await mapTransactionStreamEvent(ledgerTransaction) + + if (event && !isClosed) await onTransaction(event) + } + + if (ledgerTransactions.length < batchSize) return + } + } + + const poll = async () => { + if (pollInFlight || isClosed) return + + pollInFlight = true + try { + await streamTransactions() + } catch (err) { + const error = toError(err, "Failed to stream transactions") + serviceLogger.error({ err: error }, "Failed to stream transactions") + cleanup() + onError(error) + } finally { + pollInFlight = false + } + } + + const startPolling = () => { + interval = setInterval(() => { + poll().catch((err) => { + const error = toError(err, "Failed to stream transactions") + serviceLogger.error({ err: error }, "Failed to stream transactions") + cleanup() + onError(error) + }) + }, pollIntervalMs) + } + + ;(async () => { + if (cursor) await streamTransactions() + if (!cursor) { + cursor = + (await transactionQueries.findLatestTransactionId()) ?? + (new mongoose.Types.ObjectId().toString() as LedgerTransactionId) + } + if (!isClosed) startPolling() + })().catch((err) => { + const error = toError(err, "Failed to initialize transaction stream") + serviceLogger.error({ err: error }, "Failed to initialize transaction stream") + cleanup() + onError(error) + }) + + return { + close: cleanup, + } + } + + return { subscribeToTransactions } +} diff --git a/core/api/src/services/transactions-stream/proto/buf.gen.yaml b/core/api/src/services/transactions-stream/proto/buf.gen.yaml new file mode 100644 index 0000000000..a015a443d4 --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/buf.gen.yaml @@ -0,0 +1,14 @@ +version: v1 + +plugins: + - name: js + out: . + opt: import_style=commonjs,binary + - name: grpc + out: . + opt: grpc_js + path: grpc_tools_node_protoc_plugin + - name: ts + out: . + opt: grpc_js + path: protoc-gen-ts diff --git a/core/api/src/services/transactions-stream/proto/transactions.proto b/core/api/src/services/transactions-stream/proto/transactions.proto new file mode 100644 index 0000000000..76743b2f8c --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/transactions.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package services.transactions.v1; + +service TransactionsStream { + rpc SubscribeTransactions(SubscribeTransactionsRequest) + returns (stream TransactionEvent) {} +} + +message SubscribeTransactionsRequest { + optional string after_transaction_id = 1; +} + +enum TransactionType { + TRANSACTION_TYPE_UNSPECIFIED = 0; + SENT = 1; + RECEIVED = 2; +} + +enum SettlementViaType { + SETTLEMENT_VIA_UNSPECIFIED = 0; + LIGHTNING = 1; + INTRA_LEDGER = 2; + ONCHAIN = 3; +} + +message TransactionEvent { + string ledger_transaction_id = 1; + string wallet_id = 2; + string account_id = 3; + string payment_hash = 4; + string preimage = 5; + int64 sats_amount = 6; + int64 cents_amount = 7; + string currency = 8; + TransactionType type = 9; + SettlementViaType settlement_via = 10; + bool pending = 11; + uint64 timestamp = 12; +} diff --git a/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts b/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts new file mode 100644 index 0000000000..fda6cb5d05 --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts @@ -0,0 +1,39 @@ +// package: services.transactions.v1 +// file: transactions.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as grpc from "@grpc/grpc-js"; +import * as transactions_pb from "./transactions_pb"; + +interface ITransactionsStreamService extends grpc.ServiceDefinition { + subscribeTransactions: ITransactionsStreamService_ISubscribeTransactions; +} + +interface ITransactionsStreamService_ISubscribeTransactions extends grpc.MethodDefinition { + path: "/services.transactions.v1.TransactionsStream/SubscribeTransactions"; + requestStream: false; + responseStream: true; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} + +export const TransactionsStreamService: ITransactionsStreamService; + +export interface ITransactionsStreamServer extends grpc.UntypedServiceImplementation { + subscribeTransactions: grpc.handleServerStreamingCall; +} + +export interface ITransactionsStreamClient { + subscribeTransactions(request: transactions_pb.SubscribeTransactionsRequest, options?: Partial): grpc.ClientReadableStream; + subscribeTransactions(request: transactions_pb.SubscribeTransactionsRequest, metadata?: grpc.Metadata, options?: Partial): grpc.ClientReadableStream; +} + +export class TransactionsStreamClient extends grpc.Client implements ITransactionsStreamClient { + constructor(address: string, credentials: grpc.ChannelCredentials, options?: Partial); + public subscribeTransactions(request: transactions_pb.SubscribeTransactionsRequest, options?: Partial): grpc.ClientReadableStream; + public subscribeTransactions(request: transactions_pb.SubscribeTransactionsRequest, metadata?: grpc.Metadata, options?: Partial): grpc.ClientReadableStream; +} diff --git a/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js b/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js new file mode 100644 index 0000000000..1a2a215c63 --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js @@ -0,0 +1,44 @@ +// GENERATED CODE -- DO NOT EDIT! + +'use strict'; +var grpc = require('@grpc/grpc-js'); +var transactions_pb = require('./transactions_pb.js'); + +function serialize_services_transactions_v1_SubscribeTransactionsRequest(arg) { + if (!(arg instanceof transactions_pb.SubscribeTransactionsRequest)) { + throw new Error('Expected argument of type services.transactions.v1.SubscribeTransactionsRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_services_transactions_v1_SubscribeTransactionsRequest(buffer_arg) { + return transactions_pb.SubscribeTransactionsRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_services_transactions_v1_TransactionEvent(arg) { + if (!(arg instanceof transactions_pb.TransactionEvent)) { + throw new Error('Expected argument of type services.transactions.v1.TransactionEvent'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_services_transactions_v1_TransactionEvent(buffer_arg) { + return transactions_pb.TransactionEvent.deserializeBinary(new Uint8Array(buffer_arg)); +} + + +var TransactionsStreamService = exports.TransactionsStreamService = { + subscribeTransactions: { + path: '/services.transactions.v1.TransactionsStream/SubscribeTransactions', + requestStream: false, + responseStream: true, + requestType: transactions_pb.SubscribeTransactionsRequest, + responseType: transactions_pb.TransactionEvent, + requestSerialize: serialize_services_transactions_v1_SubscribeTransactionsRequest, + requestDeserialize: deserialize_services_transactions_v1_SubscribeTransactionsRequest, + responseSerialize: serialize_services_transactions_v1_TransactionEvent, + responseDeserialize: deserialize_services_transactions_v1_TransactionEvent, + }, +}; + +exports.TransactionsStreamClient = grpc.makeGenericClientConstructor(TransactionsStreamService, 'TransactionsStream'); diff --git a/core/api/src/services/transactions-stream/proto/transactions_pb.d.ts b/core/api/src/services/transactions-stream/proto/transactions_pb.d.ts new file mode 100644 index 0000000000..c86f55421c --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/transactions_pb.d.ts @@ -0,0 +1,96 @@ +// package: services.transactions.v1 +// file: transactions.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as jspb from "google-protobuf"; + +export class SubscribeTransactionsRequest extends jspb.Message { + + hasAfterTransactionId(): boolean; + clearAfterTransactionId(): void; + getAfterTransactionId(): string | undefined; + setAfterTransactionId(value: string): SubscribeTransactionsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SubscribeTransactionsRequest.AsObject; + static toObject(includeInstance: boolean, msg: SubscribeTransactionsRequest): SubscribeTransactionsRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: SubscribeTransactionsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SubscribeTransactionsRequest; + static deserializeBinaryFromReader(message: SubscribeTransactionsRequest, reader: jspb.BinaryReader): SubscribeTransactionsRequest; +} + +export namespace SubscribeTransactionsRequest { + export type AsObject = { + afterTransactionId?: string, + } +} + +export class TransactionEvent extends jspb.Message { + getLedgerTransactionId(): string; + setLedgerTransactionId(value: string): TransactionEvent; + getWalletId(): string; + setWalletId(value: string): TransactionEvent; + getAccountId(): string; + setAccountId(value: string): TransactionEvent; + getPaymentHash(): string; + setPaymentHash(value: string): TransactionEvent; + getPreimage(): string; + setPreimage(value: string): TransactionEvent; + getSatsAmount(): number; + setSatsAmount(value: number): TransactionEvent; + getCentsAmount(): number; + setCentsAmount(value: number): TransactionEvent; + getCurrency(): string; + setCurrency(value: string): TransactionEvent; + getType(): TransactionType; + setType(value: TransactionType): TransactionEvent; + getSettlementVia(): SettlementViaType; + setSettlementVia(value: SettlementViaType): TransactionEvent; + getPending(): boolean; + setPending(value: boolean): TransactionEvent; + getTimestamp(): number; + setTimestamp(value: number): TransactionEvent; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): TransactionEvent.AsObject; + static toObject(includeInstance: boolean, msg: TransactionEvent): TransactionEvent.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: TransactionEvent, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): TransactionEvent; + static deserializeBinaryFromReader(message: TransactionEvent, reader: jspb.BinaryReader): TransactionEvent; +} + +export namespace TransactionEvent { + export type AsObject = { + ledgerTransactionId: string, + walletId: string, + accountId: string, + paymentHash: string, + preimage: string, + satsAmount: number, + centsAmount: number, + currency: string, + type: TransactionType, + settlementVia: SettlementViaType, + pending: boolean, + timestamp: number, + } +} + +export enum TransactionType { + TRANSACTION_TYPE_UNSPECIFIED = 0, + SENT = 1, + RECEIVED = 2, +} + +export enum SettlementViaType { + SETTLEMENT_VIA_UNSPECIFIED = 0, + LIGHTNING = 1, + INTRA_LEDGER = 2, + ONCHAIN = 3, +} diff --git a/core/api/src/services/transactions-stream/proto/transactions_pb.js b/core/api/src/services/transactions-stream/proto/transactions_pb.js new file mode 100644 index 0000000000..bde369d5a6 --- /dev/null +++ b/core/api/src/services/transactions-stream/proto/transactions_pb.js @@ -0,0 +1,697 @@ +// source: transactions.proto +/** + * @fileoverview + * @enhanceable + * @suppress {missingRequire} reports error on implicit type usages. + * @suppress {messageConventions} JS Compiler reports an error if a variable or + * field starts with 'MSG_' and isn't a translatable message. + * @public + */ +// GENERATED CODE -- DO NOT EDIT! +/* eslint-disable */ +// @ts-nocheck + +var jspb = require('google-protobuf'); +var goog = jspb; +var global = + (typeof globalThis !== 'undefined' && globalThis) || + (typeof window !== 'undefined' && window) || + (typeof global !== 'undefined' && global) || + (typeof self !== 'undefined' && self) || + (function () { return this; }).call(null) || + Function('return this')(); + +goog.exportSymbol('proto.services.transactions.v1.SettlementViaType', null, global); +goog.exportSymbol('proto.services.transactions.v1.SubscribeTransactionsRequest', null, global); +goog.exportSymbol('proto.services.transactions.v1.TransactionEvent', null, global); +goog.exportSymbol('proto.services.transactions.v1.TransactionType', null, global); +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.services.transactions.v1.SubscribeTransactionsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.services.transactions.v1.SubscribeTransactionsRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.services.transactions.v1.SubscribeTransactionsRequest.displayName = 'proto.services.transactions.v1.SubscribeTransactionsRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.services.transactions.v1.TransactionEvent = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.services.transactions.v1.TransactionEvent, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.services.transactions.v1.TransactionEvent.displayName = 'proto.services.transactions.v1.TransactionEvent'; +} + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.services.transactions.v1.SubscribeTransactionsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.services.transactions.v1.SubscribeTransactionsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.toObject = function(includeInstance, msg) { + var f, obj = { +afterTransactionId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.services.transactions.v1.SubscribeTransactionsRequest} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.services.transactions.v1.SubscribeTransactionsRequest; + return proto.services.transactions.v1.SubscribeTransactionsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.services.transactions.v1.SubscribeTransactionsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.services.transactions.v1.SubscribeTransactionsRequest} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setAfterTransactionId(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.services.transactions.v1.SubscribeTransactionsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.services.transactions.v1.SubscribeTransactionsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } +}; + + +/** + * optional string after_transaction_id = 1; + * @return {string} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.getAfterTransactionId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.SubscribeTransactionsRequest} returns this + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.setAfterTransactionId = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.services.transactions.v1.SubscribeTransactionsRequest} returns this + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.clearAfterTransactionId = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.services.transactions.v1.SubscribeTransactionsRequest.prototype.hasAfterTransactionId = function() { + return jspb.Message.getField(this, 1) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.services.transactions.v1.TransactionEvent.prototype.toObject = function(opt_includeInstance) { + return proto.services.transactions.v1.TransactionEvent.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.services.transactions.v1.TransactionEvent} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.services.transactions.v1.TransactionEvent.toObject = function(includeInstance, msg) { + var f, obj = { +ledgerTransactionId: jspb.Message.getFieldWithDefault(msg, 1, ""), +walletId: jspb.Message.getFieldWithDefault(msg, 2, ""), +accountId: jspb.Message.getFieldWithDefault(msg, 3, ""), +paymentHash: jspb.Message.getFieldWithDefault(msg, 4, ""), +preimage: jspb.Message.getFieldWithDefault(msg, 5, ""), +satsAmount: jspb.Message.getFieldWithDefault(msg, 6, 0), +centsAmount: jspb.Message.getFieldWithDefault(msg, 7, 0), +currency: jspb.Message.getFieldWithDefault(msg, 8, ""), +type: jspb.Message.getFieldWithDefault(msg, 9, 0), +settlementVia: jspb.Message.getFieldWithDefault(msg, 10, 0), +pending: jspb.Message.getBooleanFieldWithDefault(msg, 11, false), +timestamp: jspb.Message.getFieldWithDefault(msg, 12, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.services.transactions.v1.TransactionEvent} + */ +proto.services.transactions.v1.TransactionEvent.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.services.transactions.v1.TransactionEvent; + return proto.services.transactions.v1.TransactionEvent.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.services.transactions.v1.TransactionEvent} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.services.transactions.v1.TransactionEvent} + */ +proto.services.transactions.v1.TransactionEvent.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setLedgerTransactionId(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setWalletId(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setAccountId(value); + break; + case 4: + var value = /** @type {string} */ (reader.readString()); + msg.setPaymentHash(value); + break; + case 5: + var value = /** @type {string} */ (reader.readString()); + msg.setPreimage(value); + break; + case 6: + var value = /** @type {number} */ (reader.readInt64()); + msg.setSatsAmount(value); + break; + case 7: + var value = /** @type {number} */ (reader.readInt64()); + msg.setCentsAmount(value); + break; + case 8: + var value = /** @type {string} */ (reader.readString()); + msg.setCurrency(value); + break; + case 9: + var value = /** @type {!proto.services.transactions.v1.TransactionType} */ (reader.readEnum()); + msg.setType(value); + break; + case 10: + var value = /** @type {!proto.services.transactions.v1.SettlementViaType} */ (reader.readEnum()); + msg.setSettlementVia(value); + break; + case 11: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setPending(value); + break; + case 12: + var value = /** @type {number} */ (reader.readUint64()); + msg.setTimestamp(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.services.transactions.v1.TransactionEvent.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.services.transactions.v1.TransactionEvent.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.services.transactions.v1.TransactionEvent} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.services.transactions.v1.TransactionEvent.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getLedgerTransactionId(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getWalletId(); + if (f.length > 0) { + writer.writeString( + 2, + f + ); + } + f = message.getAccountId(); + if (f.length > 0) { + writer.writeString( + 3, + f + ); + } + f = message.getPaymentHash(); + if (f.length > 0) { + writer.writeString( + 4, + f + ); + } + f = message.getPreimage(); + if (f.length > 0) { + writer.writeString( + 5, + f + ); + } + f = message.getSatsAmount(); + if (f !== 0) { + writer.writeInt64( + 6, + f + ); + } + f = message.getCentsAmount(); + if (f !== 0) { + writer.writeInt64( + 7, + f + ); + } + f = message.getCurrency(); + if (f.length > 0) { + writer.writeString( + 8, + f + ); + } + f = message.getType(); + if (f !== 0.0) { + writer.writeEnum( + 9, + f + ); + } + f = message.getSettlementVia(); + if (f !== 0.0) { + writer.writeEnum( + 10, + f + ); + } + f = message.getPending(); + if (f) { + writer.writeBool( + 11, + f + ); + } + f = message.getTimestamp(); + if (f !== 0) { + writer.writeUint64( + 12, + f + ); + } +}; + + +/** + * optional string ledger_transaction_id = 1; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getLedgerTransactionId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setLedgerTransactionId = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional string wallet_id = 2; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getWalletId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setWalletId = function(value) { + return jspb.Message.setProto3StringField(this, 2, value); +}; + + +/** + * optional string account_id = 3; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getAccountId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setAccountId = function(value) { + return jspb.Message.setProto3StringField(this, 3, value); +}; + + +/** + * optional string payment_hash = 4; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getPaymentHash = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setPaymentHash = function(value) { + return jspb.Message.setProto3StringField(this, 4, value); +}; + + +/** + * optional string preimage = 5; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getPreimage = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 5, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setPreimage = function(value) { + return jspb.Message.setProto3StringField(this, 5, value); +}; + + +/** + * optional int64 sats_amount = 6; + * @return {number} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getSatsAmount = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 6, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setSatsAmount = function(value) { + return jspb.Message.setProto3IntField(this, 6, value); +}; + + +/** + * optional int64 cents_amount = 7; + * @return {number} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getCentsAmount = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 7, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setCentsAmount = function(value) { + return jspb.Message.setProto3IntField(this, 7, value); +}; + + +/** + * optional string currency = 8; + * @return {string} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getCurrency = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 8, "")); +}; + + +/** + * @param {string} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setCurrency = function(value) { + return jspb.Message.setProto3StringField(this, 8, value); +}; + + +/** + * optional TransactionType type = 9; + * @return {!proto.services.transactions.v1.TransactionType} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getType = function() { + return /** @type {!proto.services.transactions.v1.TransactionType} */ (jspb.Message.getFieldWithDefault(this, 9, 0)); +}; + + +/** + * @param {!proto.services.transactions.v1.TransactionType} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setType = function(value) { + return jspb.Message.setProto3EnumField(this, 9, value); +}; + + +/** + * optional SettlementViaType settlement_via = 10; + * @return {!proto.services.transactions.v1.SettlementViaType} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getSettlementVia = function() { + return /** @type {!proto.services.transactions.v1.SettlementViaType} */ (jspb.Message.getFieldWithDefault(this, 10, 0)); +}; + + +/** + * @param {!proto.services.transactions.v1.SettlementViaType} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setSettlementVia = function(value) { + return jspb.Message.setProto3EnumField(this, 10, value); +}; + + +/** + * optional bool pending = 11; + * @return {boolean} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getPending = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 11, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setPending = function(value) { + return jspb.Message.setProto3BooleanField(this, 11, value); +}; + + +/** + * optional uint64 timestamp = 12; + * @return {number} + */ +proto.services.transactions.v1.TransactionEvent.prototype.getTimestamp = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 12, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.services.transactions.v1.TransactionEvent} returns this + */ +proto.services.transactions.v1.TransactionEvent.prototype.setTimestamp = function(value) { + return jspb.Message.setProto3IntField(this, 12, value); +}; + + +/** + * @enum {number} + */ +proto.services.transactions.v1.TransactionType = { + TRANSACTION_TYPE_UNSPECIFIED: 0, + SENT: 1, + RECEIVED: 2 +}; + +/** + * @enum {number} + */ +proto.services.transactions.v1.SettlementViaType = { + SETTLEMENT_VIA_UNSPECIFIED: 0, + LIGHTNING: 1, + INTRA_LEDGER: 2, + ONCHAIN: 3 +}; + +goog.object.extend(exports, proto.services.transactions.v1); diff --git a/flake.nix b/flake.nix index f8bb70fd76..1d63134099 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,7 @@ (import rust-overlay) ]; pkgs = import nixpkgs {inherit overlays system;}; + bufPkg = if pkgs.stdenv.isDarwin then dockerPkgs.buf else pkgs.buf; rustVersion = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; rust-toolchain = rustVersion.override { extensions = ["rust-analyzer" "rust-src"]; @@ -84,7 +85,7 @@ reindeer gitMinimal grpcurl - buf + bufPkg netcat ] ++ buck2NativeBuildInputs @@ -299,6 +300,7 @@ api-trigger = tscDerivation {pkgName = "api-trigger";}; api-ws-server = tscDerivation {pkgName = "api-ws-server";}; api-exporter = tscDerivation {pkgName = "api-exporter";}; + api-transactions-grpc-stream = tscDerivation {pkgName = "api-transactions-grpc-stream";}; api-cron = tscDerivation {pkgName = "api-cron";}; consent = nextDerivation {pkgName = "consent";}; @@ -365,6 +367,15 @@ BUCK2_VERSION = buck2Version; COMPOSE_PROJECT_NAME = "galoy-dev"; + shellHook = '' + if [ -d "$PWD/node_modules/.bin" ]; then + export PATH="$PWD/node_modules/.bin:$PATH" + fi + + if [ -d "$PWD/core/api/node_modules/.bin" ]; then + export PATH="$PWD/core/api/node_modules/.bin:$PATH" + fi + ''; }; formatter = alejandra; From d04c67ef85ba71b053bbbf461f93cb621b519f33 Mon Sep 17 00:00:00 2001 From: basarrcan Date: Tue, 21 Apr 2026 16:07:24 +0300 Subject: [PATCH 02/11] test(api): cover transactions stream replay and mapping --- .../transactions-stream/grpc-server.spec.ts | 106 +++++++ .../transactions-stream/helpers.spec.ts | 157 ++++++++++ .../transactions-stream/index.spec.ts | 268 ++++++++++++++++++ 3 files changed, 531 insertions(+) create mode 100644 core/api/test/unit/services/transactions-stream/grpc-server.spec.ts create mode 100644 core/api/test/unit/services/transactions-stream/helpers.spec.ts create mode 100644 core/api/test/unit/services/transactions-stream/index.spec.ts diff --git a/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts b/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts new file mode 100644 index 0000000000..450335b662 --- /dev/null +++ b/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts @@ -0,0 +1,106 @@ +import { EventEmitter } from "events" + +import { status } from "@grpc/grpc-js" + +import { TransactionsGrpcServer } from "@/services/transactions-stream/grpc-server" +import { SubscribeTransactionsRequest } from "@/services/transactions-stream/proto/transactions_pb" + +import { + TransactionsStreamSettlementVia, + TransactionsStreamTransactionType, +} from "@/domain/transactions-stream" +import { WalletCurrency } from "@/domain/shared" + +type MockCall = EventEmitter & { + request: SubscribeTransactionsRequest + write: jest.Mock + destroy: jest.Mock +} + +const flushMicrotasks = async () => { + await Promise.resolve() + await Promise.resolve() +} + +const createCall = (afterTransactionId?: string): MockCall => { + const request = new SubscribeTransactionsRequest() + if (afterTransactionId !== undefined) request.setAfterTransactionId(afterTransactionId) + + const call = new EventEmitter() as MockCall + call.request = request + call.write = jest.fn() + call.destroy = jest.fn() + + return call +} + +const createEvent = (): TransactionStreamEvent => ({ + ledgerTransactionId: "661111111111111111111111" as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + accountId: "account-1" as AccountId, + paymentHash: "payment-hash", + preimage: "preimage", + satsAmount: 100, + centsAmount: 200, + currency: WalletCurrency.Btc, + type: TransactionsStreamTransactionType.Received, + settlementVia: TransactionsStreamSettlementVia.Lightning, + pending: false, + timestamp: new Date("2024-01-01T00:00:05Z"), +}) + +describe("TransactionsGrpcServer", () => { + it("returns INVALID_ARGUMENT for malformed cursors", () => { + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockReturnValue(new Error("invalid cursor")), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("not-an-object-id") + + grpcServer.subscribeTransactions(call as never) + + expect(call.destroy).toHaveBeenCalledTimes(1) + expect(call.destroy.mock.calls[0][0].code).toBe(status.INVALID_ARGUMENT) + expect(transactionsStreamService.subscribeToTransactions).toHaveBeenCalledWith({ + afterTransactionId: "not-an-object-id", + onTransaction: expect.any(Function), + onError: expect.any(Function), + }) + }) + + it("maps domain events to grpc messages and closes the subscription", async () => { + const close = jest.fn() + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockImplementation(({ onTransaction }) => { + onTransaction(createEvent()) + return { close } + }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("661111111111111111111110") + + grpcServer.subscribeTransactions(call as never) + await flushMicrotasks() + + expect(call.write).toHaveBeenCalledTimes(1) + expect(call.write.mock.calls[0][0].getLedgerTransactionId()).toBe( + "661111111111111111111111", + ) + expect(call.write.mock.calls[0][0].getWalletId()).toBe("wallet-1") + expect(call.write.mock.calls[0][0].getAccountId()).toBe("account-1") + expect(call.write.mock.calls[0][0].getPaymentHash()).toBe("payment-hash") + expect(call.write.mock.calls[0][0].getPreimage()).toBe("preimage") + expect(call.write.mock.calls[0][0].getSatsAmount()).toBe(100) + expect(call.write.mock.calls[0][0].getCentsAmount()).toBe(200) + expect(call.write.mock.calls[0][0].getCurrency()).toBe(WalletCurrency.Btc) + expect(call.write.mock.calls[0][0].getTimestamp()).toBe(1704067205) + + call.emit("close") + + expect(close).toHaveBeenCalledTimes(1) + }) +}) diff --git a/core/api/test/unit/services/transactions-stream/helpers.spec.ts b/core/api/test/unit/services/transactions-stream/helpers.spec.ts new file mode 100644 index 0000000000..2035648714 --- /dev/null +++ b/core/api/test/unit/services/transactions-stream/helpers.spec.ts @@ -0,0 +1,157 @@ +import mongoose from "mongoose" + +import { + createAccountIdResolver, + createPreimageLoader, + createPreimageResolver, + createTransactionStreamEventMapper, + parseWalletId, +} from "@/services/transactions-stream/helpers" + +import { + TransactionsStreamSettlementVia, + TransactionsStreamTransactionType, + ledgerTransactionTypeToTransactionsStreamSettlementVia, +} from "@/domain/transactions-stream" +import { LedgerTransactionType } from "@/domain/ledger" +import { WalletCurrency } from "@/domain/shared" + +describe("parseWalletId", () => { + it("returns the wallet id for liabilities accounts", () => { + expect(parseWalletId("Liabilities:wallet-123")).toBe("wallet-123") + }) + + it("returns undefined for non-liabilities accounts", () => { + expect(parseWalletId("Assets:wallet-123")).toBeUndefined() + }) +}) + +describe("mapLedgerTransactionTypeToSettlementVia", () => { + it("maps ledger types to the expected settlement enum", () => { + expect( + ledgerTransactionTypeToTransactionsStreamSettlementVia( + LedgerTransactionType.Invoice, + ), + ).toBe(TransactionsStreamSettlementVia.Lightning) + expect( + ledgerTransactionTypeToTransactionsStreamSettlementVia( + LedgerTransactionType.LnIntraLedger, + ), + ).toBe(TransactionsStreamSettlementVia.Intraledger) + expect( + ledgerTransactionTypeToTransactionsStreamSettlementVia( + LedgerTransactionType.OnchainPayment, + ), + ).toBe(TransactionsStreamSettlementVia.Onchain) + expect( + ledgerTransactionTypeToTransactionsStreamSettlementVia(LedgerTransactionType.Fee), + ).toBe(TransactionsStreamSettlementVia.Unspecified) + }) +}) + +describe("createAccountIdResolver", () => { + it("caches wallet lookups", async () => { + const loadAccountId = jest.fn().mockResolvedValue("account-1") + const resolveAccountId = createAccountIdResolver({ loadAccountId }) + + await expect(resolveAccountId("wallet-1" as WalletId)).resolves.toBe("account-1") + await expect(resolveAccountId("wallet-1" as WalletId)).resolves.toBe("account-1") + + expect(loadAccountId).toHaveBeenCalledTimes(1) + }) +}) + +describe("createPreimageLoader", () => { + it("prefers transaction metadata before falling back to invoices", async () => { + const findTransactionMetadataById = jest + .fn() + .mockResolvedValue({ revealedPreImage: "revealed-preimage" }) + const findWalletInvoiceById = jest.fn() + + const loadPreimage = createPreimageLoader({ + findTransactionMetadataById, + findWalletInvoiceById, + }) + + await expect( + loadPreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + ).resolves.toBe("revealed-preimage") + expect(findWalletInvoiceById).not.toHaveBeenCalled() + }) + + it("falls back to the invoice secret and then an empty string", async () => { + const findTransactionMetadataById = jest.fn().mockResolvedValue(null) + const findWalletInvoiceById = jest + .fn() + .mockResolvedValueOnce({ secret: "invoice-secret" }) + .mockResolvedValueOnce(null) + + const loadPreimage = createPreimageLoader({ + findTransactionMetadataById, + findWalletInvoiceById, + }) + + await expect( + loadPreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + ).resolves.toBe("invoice-secret") + await expect(loadPreimage({ transactionId: "tx-2" })).resolves.toBe("") + }) +}) + +describe("createPreimageResolver", () => { + it("caches preimages by transaction id", async () => { + const loadPreimage = jest.fn().mockResolvedValue("preimage-1") + const resolvePreimage = createPreimageResolver({ loadPreimage }) + + await expect( + resolvePreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + ).resolves.toBe("preimage-1") + await expect( + resolvePreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + ).resolves.toBe("preimage-1") + + expect(loadPreimage).toHaveBeenCalledTimes(1) + }) +}) + +describe("createTransactionStreamEventMapper", () => { + it("maps ledger transactions into transaction stream events", async () => { + const resolveAccountId = jest.fn().mockResolvedValue("account-1") + const resolvePreimage = jest.fn().mockResolvedValue("preimage-1") + const { mapTransactionStreamEvent } = createTransactionStreamEventMapper({ + resolveAccountId, + resolvePreimage, + }) + + const ledgerTransaction = { + _id: new mongoose.Types.ObjectId("661111111111111111111111"), + accounts: "Liabilities:wallet-1", + hash: "payment-hash-1", + type: LedgerTransactionType.Payment, + pending: false, + currency: WalletCurrency.Btc, + satsAmount: 321, + centsAmount: 654, + credit: -321, + datetime: new Date("2024-01-01T00:00:05Z"), + timestamp: new Date("2024-01-01T00:00:00Z"), + } + + const event = await mapTransactionStreamEvent(ledgerTransaction) + + expect(event).toEqual({ + ledgerTransactionId: "661111111111111111111111", + walletId: "wallet-1", + accountId: "account-1", + paymentHash: "payment-hash-1", + preimage: "preimage-1", + satsAmount: 321, + centsAmount: 654, + currency: WalletCurrency.Btc, + type: TransactionsStreamTransactionType.Sent, + settlementVia: TransactionsStreamSettlementVia.Lightning, + pending: false, + timestamp: new Date("2024-01-01T00:00:05Z"), + }) + }) +}) diff --git a/core/api/test/unit/services/transactions-stream/index.spec.ts b/core/api/test/unit/services/transactions-stream/index.spec.ts new file mode 100644 index 0000000000..8d505a0db8 --- /dev/null +++ b/core/api/test/unit/services/transactions-stream/index.spec.ts @@ -0,0 +1,268 @@ +import mongoose from "mongoose" + +import { + TransactionsStreamService, + TransactionStreamQueries, +} from "@/services/transactions-stream" +import { TransactionStreamRecord } from "@/services/transactions-stream/helpers" + +import { + TransactionsStreamSettlementVia, + TransactionsStreamTransactionType, +} from "@/domain/transactions-stream" +import { LedgerTransactionType } from "@/domain/ledger" +import { WalletCurrency } from "@/domain/shared" + +jest.useFakeTimers() + +afterAll(() => { + jest.useRealTimers() +}) + +afterEach(() => { + jest.clearAllMocks() + jest.clearAllTimers() +}) + +const flushMicrotasks = async () => { + await Promise.resolve() + await Promise.resolve() +} + +const createLedgerTransaction = ( + id: string, + overrides: Partial = {}, +): TransactionStreamRecord => + ({ + _id: new mongoose.Types.ObjectId(id), + accounts: "Liabilities:wallet-1", + hash: "payment-hash", + type: LedgerTransactionType.Invoice, + pending: false, + currency: WalletCurrency.Btc, + satsAmount: 100, + centsAmount: 200, + credit: 100, + datetime: new Date("2024-01-01T00:00:00Z"), + timestamp: new Date("2024-01-01T00:00:00Z"), + ...overrides, + }) as TransactionStreamRecord + +const createEvent = (ledgerTransactionId: string): TransactionStreamEvent => ({ + ledgerTransactionId: ledgerTransactionId as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + accountId: "account-1" as AccountId, + paymentHash: "payment-hash", + preimage: "preimage", + satsAmount: 100, + centsAmount: 200, + currency: WalletCurrency.Btc, + type: TransactionsStreamTransactionType.Received, + settlementVia: TransactionsStreamSettlementVia.Lightning, + pending: false, + timestamp: new Date("2024-01-01T00:00:00Z"), +}) + +const subscribe = ({ + service, + afterTransactionId, +}: { + service: ReturnType + afterTransactionId?: string +}) => { + const onTransaction = jest.fn() + const onError = jest.fn() + const subscription = service.subscribeToTransactions({ + afterTransactionId, + onTransaction, + onError, + }) + + return { onTransaction, onError, subscription } +} + +describe("TransactionsStreamService", () => { + it("returns an error for malformed cursors", () => { + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter: jest.fn(), + findLatestTransactionId: jest.fn(), + } + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent: jest.fn(), + }) + + const { subscription } = subscribe({ + service, + afterTransactionId: "not-an-object-id", + }) + + expect(subscription).toBeInstanceOf(Error) + expect(transactionQueries.listSettledTransactionsAfter).not.toHaveBeenCalled() + }) + + it("replays transactions when after_transaction_id is provided", async () => { + const replayTxn = createLedgerTransaction("661111111111111111111112") + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter: jest.fn().mockResolvedValue([replayTxn]), + findLatestTransactionId: jest.fn(), + } + const mapTransactionStreamEvent = jest + .fn() + .mockResolvedValue(createEvent(replayTxn._id.toString())) + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent, + pollIntervalMs: 200, + }) + + const { onTransaction } = subscribe({ + service, + afterTransactionId: "661111111111111111111111", + }) + await flushMicrotasks() + + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledWith({ + afterTransactionId: "661111111111111111111111", + limit: 100, + }) + expect(onTransaction).toHaveBeenCalledTimes(1) + expect(onTransaction).toHaveBeenCalledWith(createEvent("661111111111111111111112")) + expect(transactionQueries.findLatestTransactionId).not.toHaveBeenCalled() + }) + + it("starts from the current tip when no cursor is provided", async () => { + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter: jest.fn().mockResolvedValue([]), + findLatestTransactionId: jest + .fn() + .mockResolvedValue("661111111111111111111111" as LedgerTransactionId), + } + const mapTransactionStreamEvent = jest.fn() + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent, + pollIntervalMs: 200, + }) + + const { onTransaction } = subscribe({ service }) + await flushMicrotasks() + + expect(transactionQueries.listSettledTransactionsAfter).not.toHaveBeenCalled() + expect(mapTransactionStreamEvent).not.toHaveBeenCalled() + expect(onTransaction).not.toHaveBeenCalled() + + await jest.advanceTimersByTimeAsync(200) + + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledWith({ + afterTransactionId: "661111111111111111111111", + limit: 100, + }) + }) + + it("polls new transactions after replay", async () => { + const replayTxn = createLedgerTransaction("661111111111111111111112") + const liveTxn = createLedgerTransaction("661111111111111111111113") + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter: jest + .fn() + .mockResolvedValueOnce([replayTxn]) + .mockResolvedValueOnce([liveTxn]), + findLatestTransactionId: jest.fn(), + } + const mapTransactionStreamEvent = jest + .fn() + .mockImplementation(async (txn: TransactionStreamRecord) => + createEvent(txn._id.toString()), + ) + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent, + pollIntervalMs: 200, + }) + + const { onTransaction } = subscribe({ + service, + afterTransactionId: "661111111111111111111111", + }) + await flushMicrotasks() + await jest.advanceTimersByTimeAsync(200) + + expect(onTransaction).toHaveBeenCalledTimes(2) + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenNthCalledWith(2, { + afterTransactionId: "661111111111111111111112", + limit: 100, + }) + expect(onTransaction).toHaveBeenNthCalledWith( + 2, + createEvent("661111111111111111111113"), + ) + }) + + it("does not overlap polling cycles", async () => { + const listSettledTransactionsAfter = jest.fn< + Promise, + [{ afterTransactionId: LedgerTransactionId; limit: number }] + >() + const findLatestTransactionId = jest + .fn, []>() + .mockResolvedValue("661111111111111111111111" as LedgerTransactionId) + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter, + findLatestTransactionId, + } + let resolveFirstPoll: ((value: TransactionStreamRecord[]) => void) | undefined + const firstPoll = new Promise((resolve) => { + resolveFirstPoll = resolve + }) + listSettledTransactionsAfter.mockImplementationOnce(() => firstPoll) + listSettledTransactionsAfter.mockResolvedValueOnce([]) + + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent: jest.fn(), + pollIntervalMs: 200, + }) + + subscribe({ service }) + await flushMicrotasks() + + await jest.advanceTimersByTimeAsync(600) + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) + + expect(resolveFirstPoll).toBeDefined() + resolveFirstPoll!([]) + await flushMicrotasks() + await jest.advanceTimersByTimeAsync(200) + + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(2) + }) + + it("stops polling when the subscription is closed", async () => { + const transactionQueries: TransactionStreamQueries = { + listSettledTransactionsAfter: jest.fn().mockResolvedValue([]), + findLatestTransactionId: jest + .fn() + .mockResolvedValue("661111111111111111111111" as LedgerTransactionId), + } + const service = TransactionsStreamService({ + transactionQueries, + mapTransactionStreamEvent: jest.fn(), + pollIntervalMs: 200, + }) + + const { subscription } = subscribe({ service }) + await flushMicrotasks() + await jest.advanceTimersByTimeAsync(200) + + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) + + expect(subscription).not.toBeInstanceOf(Error) + if (subscription instanceof Error) return + + subscription.close() + await jest.advanceTimersByTimeAsync(400) + + expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) + }) +}) From c83342e2bd850c16c5a8380c99e7b9a29f0e296d Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 30 Apr 2026 05:24:42 +0300 Subject: [PATCH 03/11] feat(api): stream settled transactions from ledger --- core/api/src/config/env.ts | 7 - core/api/src/config/index.ts | 2 - core/api/src/domain/ledger/index.types.d.ts | 9 + core/api/src/services/ledger/index.ts | 6 + .../ledger/stream-settled-transactions.ts | 190 ++++++++++++ .../services/transactions-stream/helpers.ts | 84 ++--- .../src/services/transactions-stream/index.ts | 150 ++------- .../stream-settled-transactions.spec.ts | 265 ++++++++++++++++ .../transactions-stream/helpers.spec.ts | 160 ++++++++-- .../transactions-stream/index.spec.ts | 291 ++++++++++-------- 10 files changed, 806 insertions(+), 358 deletions(-) create mode 100644 core/api/src/services/ledger/stream-settled-transactions.ts create mode 100644 core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts diff --git a/core/api/src/config/env.ts b/core/api/src/config/env.ts index 23be8c1be5..a64e4f5e0a 100644 --- a/core/api/src/config/env.ts +++ b/core/api/src/config/env.ts @@ -40,11 +40,6 @@ export const env = createEnv({ .or(z.string()) .pipe(z.coerce.number()) .default(8889), - TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS: z - .number() - .or(z.string()) - .pipe(z.coerce.number()) - .default(200), KRATOS_PG_CON: z.string().url(), OATHKEEPER_DECISION_ENDPOINT: z.string().url(), @@ -187,8 +182,6 @@ export const env = createEnv({ TRANSACTIONS_GRPC_STREAM_PORT: process.env.TRANSACTIONS_GRPC_STREAM_PORT, TRANSACTIONS_GRPC_STREAM_HEALTH_PORT: process.env.TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, - TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS: - process.env.TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS, KRATOS_PG_CON: process.env.KRATOS_PG_CON, OATHKEEPER_DECISION_ENDPOINT: process.env.OATHKEEPER_DECISION_ENDPOINT, diff --git a/core/api/src/config/index.ts b/core/api/src/config/index.ts index 191d471249..361e7ecbb9 100644 --- a/core/api/src/config/index.ts +++ b/core/api/src/config/index.ts @@ -151,8 +151,6 @@ export const WEBSOCKET_PORT = env.WEBSOCKET_PORT export const TRANSACTIONS_GRPC_STREAM_PORT = env.TRANSACTIONS_GRPC_STREAM_PORT export const TRANSACTIONS_GRPC_STREAM_HEALTH_PORT = env.TRANSACTIONS_GRPC_STREAM_HEALTH_PORT -export const TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS = - env.TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS export const KRATOS_PG_CON = env.KRATOS_PG_CON export const OATHKEEPER_DECISION_ENDPOINT = env.OATHKEEPER_DECISION_ENDPOINT export const GALOY_API_PORT = env.GALOY_API_PORT diff --git a/core/api/src/domain/ledger/index.types.d.ts b/core/api/src/domain/ledger/index.types.d.ts index 6b674d529c..0127099637 100644 --- a/core/api/src/domain/ledger/index.types.d.ts +++ b/core/api/src/domain/ledger/index.types.d.ts @@ -240,6 +240,11 @@ type IsOnChainReceiptTxRecordedForWalletResult = { newAddressRequestId: OnChainAddressRequestId | undefined } +type StreamSettledTransactionsArgs = { + afterTransactionId?: LedgerTransactionId + signal?: AbortSignal +} + interface ILedgerService { updateMetadataByHash( ledgerTxMetadata: @@ -303,6 +308,10 @@ interface ILedgerService { listAllPaymentHashes(): AsyncGenerator + streamSettledTransactions( + args?: StreamSettledTransactionsArgs, + ): AsyncGenerator | LedgerError> + getPendingPaymentsCount(walletId: WalletId): Promise getWalletBalance(walletId: WalletId): Promise diff --git a/core/api/src/services/ledger/index.ts b/core/api/src/services/ledger/index.ts index 5361d52852..6bd0c3db13 100644 --- a/core/api/src/services/ledger/index.ts +++ b/core/api/src/services/ledger/index.ts @@ -16,6 +16,7 @@ import * as caching from "./caching" import { TransactionsMetadataRepository } from "./services" import { send } from "./send" +import { createStreamSettledTransactions } from "./stream-settled-transactions" import { translateToLedgerTx, translateToLedgerTxWithMetadata } from "./translate" @@ -296,6 +297,10 @@ export const LedgerService = (): ILedgerService => { } } + const streamSettledTransactions = createStreamSettledTransactions({ + transactionModel: Transaction, + }) + const getPendingPaymentsCount = async ( walletId: WalletId, ): Promise => { @@ -537,6 +542,7 @@ export const LedgerService = (): ILedgerService => { getTransactionsByWalletIdAndContactUsername, listPendingPayments, listAllPaymentHashes, + streamSettledTransactions, getPendingPaymentsCount, getWalletBalance, getWalletBalanceAmount, diff --git a/core/api/src/services/ledger/stream-settled-transactions.ts b/core/api/src/services/ledger/stream-settled-transactions.ts new file mode 100644 index 0000000000..f4478ce674 --- /dev/null +++ b/core/api/src/services/ledger/stream-settled-transactions.ts @@ -0,0 +1,190 @@ +import { translateToLedgerTx } from "./translate" + +import { LedgerTransactionType } from "@/domain/ledger" +import { UnknownLedgerError } from "@/domain/ledger/errors" +import { WalletCurrency } from "@/domain/shared" +import { toObjectId } from "@/services/mongoose/utils" + +const SETTLED_TRANSACTION_STREAM_BATCH_SIZE = 100 +const DEFAULT_REPLAY_DEDUPE_CACHE_SIZE = 10_000 +const LIABILITIES_ACCOUNT_PATTERN = /^Liabilities:/ +const EXCLUDED_SETTLED_TRANSACTION_TYPES: LedgerTransactionType[] = [ + LedgerTransactionType.Fee, + LedgerTransactionType.ToColdStorage, + LedgerTransactionType.ToHotWallet, + LedgerTransactionType.Escrow, + LedgerTransactionType.RoutingRevenue, + LedgerTransactionType.Reconciliation, +] +const SETTLED_TRANSACTION_CHANGE_OPERATIONS = ["insert", "replace", "update"] +const SETTLED_TRANSACTION_CHANGE_OPERATION_MATCH = [ + { operationType: { $in: ["insert", "replace"] } }, + { "updateDescription.updatedFields.pending": false }, +] + +type TransactionCursor = AsyncIterable & { + close: () => Promise +} + +type TransactionFindQuery = { + sort: (sort: Record) => { + cursor: (options: { batchSize: number }) => TransactionCursor + } +} + +type TransactionChangeStream = { + next: () => Promise + close: () => Promise +} + +type SettledTransactionModel = { + find: (filter: Record) => TransactionFindQuery + watch: ( + pipeline: Record[], + options: { fullDocument: "updateLookup" }, + ) => TransactionChangeStream +} + +type StreamSettledTransactionsConfig = { + transactionModel: SettledTransactionModel + translateLedgerTransaction?: ( + tx: ILedgerTransaction, + ) => LedgerTransaction + maxReplayDedupeCacheSize?: number +} + +export const settledTransactionFilter = ( + afterTransactionId?: LedgerTransactionId, +): Record => { + const filter: Record = { + accounts: LIABILITIES_ACCOUNT_PATTERN, + pending: false, + voided: { $ne: true }, + type: { $nin: EXCLUDED_SETTLED_TRANSACTION_TYPES }, + } + + if (afterTransactionId) { + filter._id = { $gt: toObjectId(afterTransactionId) } + } + + return filter +} + +export const settledTransactionChangeStreamPipeline = ( + afterTransactionId?: LedgerTransactionId, +): Record[] => { + const match: Record = { + "operationType": { $in: SETTLED_TRANSACTION_CHANGE_OPERATIONS }, + "$or": SETTLED_TRANSACTION_CHANGE_OPERATION_MATCH, + "fullDocument.accounts": LIABILITIES_ACCOUNT_PATTERN, + "fullDocument.pending": false, + "fullDocument.voided": { $ne: true }, + "fullDocument.type": { $nin: EXCLUDED_SETTLED_TRANSACTION_TYPES }, + } + + if (afterTransactionId) { + match["fullDocument._id"] = { + $gt: toObjectId(afterTransactionId), + } + } + + return [{ $match: match }] +} + +export const createStreamSettledTransactions = ({ + transactionModel, + translateLedgerTransaction = translateToLedgerTx, + maxReplayDedupeCacheSize = DEFAULT_REPLAY_DEDUPE_CACHE_SIZE, +}: StreamSettledTransactionsConfig) => { + async function* streamSettledTransactions({ + afterTransactionId, + signal, + }: StreamSettledTransactionsArgs = {}): AsyncGenerator< + LedgerTransaction | LedgerError + > { + const replayedTransactionIds = new Set() + const trackReplayedTransactionId = (id: LedgerTransactionId) => { + if (maxReplayDedupeCacheSize <= 0) return + + replayedTransactionIds.delete(id) + replayedTransactionIds.add(id) + + if (replayedTransactionIds.size <= maxReplayDedupeCacheSize) return + + const oldestReplayedTransactionId = replayedTransactionIds.values().next().value + if (oldestReplayedTransactionId) { + replayedTransactionIds.delete(oldestReplayedTransactionId) + } + } + const changeStream = transactionModel.watch( + settledTransactionChangeStreamPipeline(afterTransactionId), + { fullDocument: "updateLookup" }, + ) + + const closeChangeStream = () => { + changeStream.close().catch(() => undefined) + } + + signal?.addEventListener("abort", closeChangeStream, { once: true }) + + const waitForNextChange = () => + changeStream.next().then( + (change) => ({ change }), + (err: unknown) => ({ err }), + ) + + let nextChange = waitForNextChange() + + try { + if (afterTransactionId) { + const cursor = transactionModel + .find(settledTransactionFilter(afterTransactionId)) + .sort({ _id: 1 }) + .cursor({ batchSize: SETTLED_TRANSACTION_STREAM_BATCH_SIZE }) + + const closeCursor = () => { + cursor.close().catch(() => undefined) + } + + signal?.addEventListener("abort", closeCursor, { once: true }) + + try { + for await (const tx of cursor) { + if (signal?.aborted) return + + const ledgerTx = translateLedgerTransaction(tx) + trackReplayedTransactionId(ledgerTx.id) + yield ledgerTx + } + } finally { + signal?.removeEventListener("abort", closeCursor) + await cursor.close().catch(() => undefined) + } + } + + while (!signal?.aborted) { + const changeResult = await nextChange + nextChange = waitForNextChange() + if ("err" in changeResult) throw changeResult.err + + const tx = (changeResult.change as { fullDocument?: ILedgerTransaction }) + .fullDocument + if (!tx) continue + + const ledgerTx = translateLedgerTransaction(tx) + if (replayedTransactionIds.delete(ledgerTx.id)) continue + + yield ledgerTx + } + } catch (err) { + if (signal?.aborted) return + + yield new UnknownLedgerError(err) + } finally { + signal?.removeEventListener("abort", closeChangeStream) + await changeStream.close().catch(() => undefined) + } + } + + return streamSettledTransactions +} diff --git a/core/api/src/services/transactions-stream/helpers.ts b/core/api/src/services/transactions-stream/helpers.ts index 2ac6c0b58d..4561ba350a 100644 --- a/core/api/src/services/transactions-stream/helpers.ts +++ b/core/api/src/services/transactions-stream/helpers.ts @@ -2,11 +2,6 @@ import { ledgerTransactionCreditToTransactionsStreamTransactionType, ledgerTransactionTypeToTransactionsStreamSettlementVia, } from "@/domain/transactions-stream" -import { - LedgerTransactionType, - liabilitiesMainAccount, - toWalletId, -} from "@/domain/ledger" import { TransactionMetadata } from "@/services/ledger/schema" import { WalletInvoice } from "@/services/mongoose/schema" @@ -15,48 +10,14 @@ import { WalletsRepository } from "@/services/mongoose/wallets" const PREIMAGE_CACHE_TTL_MS = 5 * 60 * 1000 const PREIMAGE_CACHE_MAX_SIZE = 10_000 -export const LIABILITIES_ACCOUNT_PREFIX = `${liabilitiesMainAccount}:` -export const LIABILITIES_ACCOUNT_PATTERN = /^Liabilities:/ - -export const EXCLUDED_LEDGER_TRANSACTION_TYPES: LedgerTransactionType[] = [ - LedgerTransactionType.Fee, - LedgerTransactionType.ToColdStorage, - LedgerTransactionType.ToHotWallet, - LedgerTransactionType.Escrow, - LedgerTransactionType.RoutingRevenue, - LedgerTransactionType.Reconciliation, -] - -export const SETTLED_TRANSACTION_FILTER = { - accounts: LIABILITIES_ACCOUNT_PATTERN, - pending: false, - type: { $nin: EXCLUDED_LEDGER_TRANSACTION_TYPES }, -} as const - type TimestampedValue = { value: T expiresAt: number } type PreimageLoaderArgs = { - transactionId: string - paymentHash?: string -} - -export type TransactionStreamRecord = Pick< - ILedgerTransaction, - | "accounts" - | "hash" - | "type" - | "pending" - | "currency" - | "satsAmount" - | "centsAmount" - | "credit" - | "datetime" - | "timestamp" -> & { - _id: ObjectId + transactionId: LedgerTransactionId + paymentHash?: PaymentHash } export type AccountIdLoader = (walletId: WalletId) => Promise @@ -68,7 +29,7 @@ type FindWalletById = ( walletId: WalletId, ) => Promise<{ accountId: AccountId } | Error | undefined> type FindTransactionMetadataById = ( - transactionId: string, + transactionId: LedgerTransactionId, ) => Promise | null | undefined> type FindWalletInvoiceById = ( paymentHash: string, @@ -109,27 +70,25 @@ class ExpiringCache { } } -export const parseWalletId = (accounts: string): WalletId | undefined => { - if (!accounts.startsWith(LIABILITIES_ACCOUNT_PREFIX)) return undefined - return toWalletId(accounts as LiabilitiesWalletId) +const defaultFindWalletById: FindWalletById = async (walletId) => { + return WalletsRepository().findById(walletId) } -const defaultFindWalletById: FindWalletById = async (walletId) => - WalletsRepository().findById(walletId) - const defaultFindTransactionMetadataById: FindTransactionMetadataById = async ( transactionId, -) => - (await TransactionMetadata.findById(transactionId).lean()) as Pick< +) => { + return (await TransactionMetadata.findById(transactionId).lean()) as Pick< TransactionMetadataRecord, "revealedPreImage" > | null +} -const defaultFindWalletInvoiceById: FindWalletInvoiceById = async (paymentHash) => - (await WalletInvoice.findById(paymentHash).lean()) as Pick< +const defaultFindWalletInvoiceById: FindWalletInvoiceById = async (paymentHash) => { + return (await WalletInvoice.findById(paymentHash).lean()) as Pick< WalletInvoiceRecord, "secret" > | null +} export const createAccountIdLoader = ({ findWalletById = defaultFindWalletById, @@ -138,7 +97,8 @@ export const createAccountIdLoader = ({ } = {}): AccountIdLoader => { return async (walletId) => { const wallet = await findWalletById(walletId) - if (!wallet || wallet instanceof Error) return undefined + if (wallet instanceof Error) throw wallet + if (!wallet) return undefined return wallet.accountId } @@ -174,7 +134,7 @@ export const createAccountIdResolver = ({ } const accountId = await loadAccountId(walletId) - walletToAccountCache.set(walletId, accountId) + if (accountId !== undefined) walletToAccountCache.set(walletId, accountId) return accountId } @@ -211,24 +171,24 @@ export const createTransactionStreamEventMapper = ({ resolvePreimage?: PreimageResolver } = {}) => { const mapTransactionStreamEvent = async ( - ledgerTransaction: TransactionStreamRecord, + ledgerTransaction: LedgerTransaction, ): Promise => { - const walletId = parseWalletId(ledgerTransaction.accounts) + const walletId = ledgerTransaction.walletId if (!walletId) return undefined const [accountId, preimage] = await Promise.all([ resolveAccountId(walletId), resolvePreimage({ - transactionId: ledgerTransaction._id.toString(), - paymentHash: ledgerTransaction.hash, + transactionId: ledgerTransaction.id, + paymentHash: ledgerTransaction.paymentHash, }), ]) return { - ledgerTransactionId: ledgerTransaction._id.toString() as LedgerTransactionId, + ledgerTransactionId: ledgerTransaction.id, walletId, accountId, - paymentHash: ledgerTransaction.hash ?? undefined, + paymentHash: ledgerTransaction.paymentHash, preimage, satsAmount: ledgerTransaction.satsAmount ?? 0, centsAmount: ledgerTransaction.centsAmount ?? 0, @@ -239,8 +199,8 @@ export const createTransactionStreamEventMapper = ({ settlementVia: ledgerTransactionTypeToTransactionsStreamSettlementVia( ledgerTransaction.type, ), - pending: Boolean(ledgerTransaction.pending), - timestamp: ledgerTransaction.datetime ?? ledgerTransaction.timestamp ?? undefined, + pending: ledgerTransaction.pendingConfirmation, + timestamp: ledgerTransaction.timestamp, } } diff --git a/core/api/src/services/transactions-stream/index.ts b/core/api/src/services/transactions-stream/index.ts index f4ac1c808d..b2efff4d44 100644 --- a/core/api/src/services/transactions-stream/index.ts +++ b/core/api/src/services/transactions-stream/index.ts @@ -1,48 +1,17 @@ -import mongoose from "mongoose" +import { createTransactionStreamEventMapper } from "./helpers" -import { - createTransactionStreamEventMapper, - SETTLED_TRANSACTION_FILTER, - TransactionStreamRecord, -} from "./helpers" - -import { TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS } from "@/config" import { checkedToLedgerTransactionId } from "@/domain/ledger" -import { Transaction } from "@/services/ledger/schema" +import { LedgerService } from "@/services/ledger" import { baseLogger } from "@/services/logger" const logger = baseLogger.child({ module: "transactions-stream" }) -const DEFAULT_BATCH_SIZE = 100 - -const TRANSACTION_STREAM_PROJECTION = { - _id: 1, - accounts: 1, - hash: 1, - type: 1, - pending: 1, - currency: 1, - satsAmount: 1, - centsAmount: 1, - credit: 1, - datetime: 1, - timestamp: 1, -} as const - const toError = (err: unknown, fallbackMessage: string): Error => { if (err instanceof Error) return err return new Error(fallbackMessage) } -export type TransactionStreamQueries = { - listSettledTransactionsAfter: (args: { - afterTransactionId: LedgerTransactionId - limit: number - }) => Promise - findLatestTransactionId: () => Promise -} - type SubscribeToTransactionsArgs = { afterTransactionId?: string onTransaction: (event: TransactionStreamEvent) => void | Promise @@ -54,53 +23,17 @@ export type TransactionsStreamSubscription = { } type TransactionsStreamServiceConfig = { - batchSize?: number - pollIntervalMs?: number - transactionQueries?: TransactionStreamQueries + ledgerService?: Pick mapTransactionStreamEvent?: ( - ledgerTransaction: TransactionStreamRecord, + ledgerTransaction: LedgerTransaction, ) => Promise logger?: Logger } -export const createTransactionStreamQueries = (): TransactionStreamQueries => { - const listSettledTransactionsAfter = async ({ - afterTransactionId, - limit, - }: { - afterTransactionId: LedgerTransactionId - limit: number - }): Promise => { - return (await Transaction.find( - { - _id: { $gt: new mongoose.Types.ObjectId(afterTransactionId) }, - ...SETTLED_TRANSACTION_FILTER, - }, - TRANSACTION_STREAM_PROJECTION, - ) - .sort({ _id: 1 }) - .limit(limit) - .lean()) as TransactionStreamRecord[] - } - - const findLatestTransactionId = async (): Promise => { - const latest = (await Transaction.findOne({}, { _id: 1 }) - .sort({ _id: -1 }) - .lean()) as { _id?: ObjectId } | null - - return latest?._id?.toString() as LedgerTransactionId | undefined - } - - return { - listSettledTransactionsAfter, - findLatestTransactionId, - } -} - const parseAfterTransactionId = ( afterTransactionId?: string, ): LedgerTransactionId | Error | undefined => { - if (!afterTransactionId) return undefined + if (afterTransactionId === undefined) return undefined const checkedLedgerTransactionId = checkedToLedgerTransactionId(afterTransactionId) if (checkedLedgerTransactionId instanceof Error) return checkedLedgerTransactionId @@ -109,9 +42,7 @@ const parseAfterTransactionId = ( } export const TransactionsStreamService = ({ - batchSize = DEFAULT_BATCH_SIZE, - pollIntervalMs = TRANSACTIONS_GRPC_STREAM_POLL_INTERVAL_MS, - transactionQueries = createTransactionStreamQueries(), + ledgerService = LedgerService(), mapTransactionStreamEvent = createTransactionStreamEventMapper() .mapTransactionStreamEvent, logger: serviceLogger = logger, @@ -124,43 +55,32 @@ export const TransactionsStreamService = ({ const parsedCursor = parseAfterTransactionId(afterTransactionId) if (parsedCursor instanceof Error) return parsedCursor - let cursor = parsedCursor let isClosed = false - let pollInFlight = false - let interval: NodeJS.Timeout | undefined + const abortController = new AbortController() + const ledgerTransactions = ledgerService.streamSettledTransactions({ + afterTransactionId: parsedCursor, + signal: abortController.signal, + }) const cleanup = () => { - if (interval) clearInterval(interval) - interval = undefined + if (isClosed) return isClosed = true + abortController.abort() + ledgerTransactions.return(undefined).catch(() => undefined) } const streamTransactions = async () => { - if (!cursor) return - - while (!isClosed) { - const ledgerTransactions = await transactionQueries.listSettledTransactionsAfter({ - afterTransactionId: cursor, - limit: batchSize, - }) - - if (ledgerTransactions.length === 0) return - - for (const ledgerTransaction of ledgerTransactions) { - cursor = ledgerTransaction._id.toString() as LedgerTransactionId - const event = await mapTransactionStreamEvent(ledgerTransaction) + for await (const ledgerTransaction of ledgerTransactions) { + if (isClosed) return + if (ledgerTransaction instanceof Error) throw ledgerTransaction - if (event && !isClosed) await onTransaction(event) - } + const event = await mapTransactionStreamEvent(ledgerTransaction) - if (ledgerTransactions.length < batchSize) return + if (event && !isClosed) await onTransaction(event) } } - const poll = async () => { - if (pollInFlight || isClosed) return - - pollInFlight = true + ;(async () => { try { await streamTransactions() } catch (err) { @@ -168,36 +88,8 @@ export const TransactionsStreamService = ({ serviceLogger.error({ err: error }, "Failed to stream transactions") cleanup() onError(error) - } finally { - pollInFlight = false } - } - - const startPolling = () => { - interval = setInterval(() => { - poll().catch((err) => { - const error = toError(err, "Failed to stream transactions") - serviceLogger.error({ err: error }, "Failed to stream transactions") - cleanup() - onError(error) - }) - }, pollIntervalMs) - } - - ;(async () => { - if (cursor) await streamTransactions() - if (!cursor) { - cursor = - (await transactionQueries.findLatestTransactionId()) ?? - (new mongoose.Types.ObjectId().toString() as LedgerTransactionId) - } - if (!isClosed) startPolling() - })().catch((err) => { - const error = toError(err, "Failed to initialize transaction stream") - serviceLogger.error({ err: error }, "Failed to initialize transaction stream") - cleanup() - onError(error) - }) + })() return { close: cleanup, diff --git a/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts b/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts new file mode 100644 index 0000000000..ad3f5a62bc --- /dev/null +++ b/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts @@ -0,0 +1,265 @@ +import mongoose from "mongoose" + +import { + createStreamSettledTransactions, + settledTransactionChangeStreamPipeline, + settledTransactionFilter, +} from "@/services/ledger/stream-settled-transactions" + +import { LedgerTransactionType } from "@/domain/ledger" +import { UnknownLedgerError } from "@/domain/ledger/errors" +import { WalletCurrency } from "@/domain/shared" + +const createRawTransaction = ( + id: string, + overrides: Partial = {}, +): ILedgerTransaction => + ({ + _id: new mongoose.Types.ObjectId(id), + credit: 100, + debit: 0, + datetime: new Date("2024-01-01T00:00:05Z"), + account_path: ["Liabilities"], + accounts: "Liabilities:wallet-1", + book: "MainBook", + memo: "", + _journal: new mongoose.Types.ObjectId("661111111111111111111199"), + timestamp: new Date("2024-01-01T00:00:00Z"), + hash: "payment-hash", + type: LedgerTransactionType.Invoice, + pending: false, + currency: WalletCurrency.Btc, + feeKnownInAdvance: false, + ...overrides, + }) as ILedgerTransaction + +const createCursor = (values: ILedgerTransaction[]) => { + const cursor = { + async *[Symbol.asyncIterator]() { + for (const value of values) yield value + }, + close: jest.fn().mockResolvedValue(undefined), + } + + return cursor +} + +const createTransactionModel = ({ + replay = [], + changes = [], + changeError, +}: { + replay?: ILedgerTransaction[] + changes?: ILedgerTransaction[] + changeError?: Error +} = {}) => { + const cursor = createCursor(replay) + const cursorFn = jest.fn().mockReturnValue(cursor) + const sort = jest.fn().mockReturnValue({ cursor: cursorFn }) + const find = jest.fn().mockReturnValue({ sort }) + + const next = jest.fn() + for (const change of changes) { + next.mockResolvedValueOnce({ fullDocument: change }) + } + if (changeError) { + next.mockRejectedValueOnce(changeError) + } else { + next.mockImplementation( + () => + new Promise(() => { + // Intentionally left pending until the stream is closed. + }), + ) + } + + const changeStream = { + next, + close: jest.fn().mockResolvedValue(undefined), + } + const watch = jest.fn().mockReturnValue(changeStream) + + return { + transactionModel: { find, watch }, + changeStream, + cursor, + cursorFn, + find, + sort, + watch, + } +} + +describe("settled transaction stream query helpers", () => { + it("filters settled customer transactions after a cursor", () => { + const filter = settledTransactionFilter( + "661111111111111111111111" as LedgerTransactionId, + ) + + expect(filter).toMatchObject({ + accounts: /^Liabilities:/, + pending: false, + voided: { $ne: true }, + type: { + $nin: [ + LedgerTransactionType.Fee, + LedgerTransactionType.ToColdStorage, + LedgerTransactionType.ToHotWallet, + LedgerTransactionType.Escrow, + LedgerTransactionType.RoutingRevenue, + LedgerTransactionType.Reconciliation, + ], + }, + }) + expect((filter._id as { $gt: mongoose.Types.ObjectId }).$gt.toString()).toBe( + "661111111111111111111111", + ) + }) + + it("builds the same constraints for live change streams", () => { + const pipeline = settledTransactionChangeStreamPipeline( + "661111111111111111111111" as LedgerTransactionId, + ) + + expect(pipeline).toEqual([ + { + $match: { + "operationType": { $in: ["insert", "replace", "update"] }, + "$or": [ + { operationType: { $in: ["insert", "replace"] } }, + { "updateDescription.updatedFields.pending": false }, + ], + "fullDocument.accounts": /^Liabilities:/, + "fullDocument.pending": false, + "fullDocument.voided": { $ne: true }, + "fullDocument.type": { + $nin: [ + LedgerTransactionType.Fee, + LedgerTransactionType.ToColdStorage, + LedgerTransactionType.ToHotWallet, + LedgerTransactionType.Escrow, + LedgerTransactionType.RoutingRevenue, + LedgerTransactionType.Reconciliation, + ], + }, + "fullDocument._id": { + $gt: new mongoose.Types.ObjectId("661111111111111111111111"), + }, + }, + }, + ]) + }) +}) + +describe("createStreamSettledTransactions", () => { + it("opens the change stream before replay and skips replay overlap", async () => { + const replayTxn = createRawTransaction("661111111111111111111112") + const liveTxn = createRawTransaction("661111111111111111111113") + const model = createTransactionModel({ + replay: [replayTxn], + changes: [replayTxn, liveTxn], + }) + const streamSettledTransactions = createStreamSettledTransactions({ + transactionModel: model.transactionModel, + }) + + const stream = streamSettledTransactions({ + afterTransactionId: "661111111111111111111111" as LedgerTransactionId, + }) + + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111112" }, + done: false, + }) + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111113" }, + done: false, + }) + await stream.return(undefined) + + expect(model.watch.mock.invocationCallOrder[0]).toBeLessThan( + model.find.mock.invocationCallOrder[0], + ) + expect(model.find).toHaveBeenCalledTimes(1) + expect(model.sort).toHaveBeenCalledWith({ _id: 1 }) + expect(model.cursorFn).toHaveBeenCalledWith({ batchSize: 100 }) + expect(model.cursor.close).toHaveBeenCalled() + expect(model.changeStream.close).toHaveBeenCalled() + }) + + it("bounds replay overlap dedupe while still skipping recent overlap", async () => { + const replayedThenEvictedTxn = createRawTransaction("661111111111111111111112") + const replayedRecentTxn = createRawTransaction("661111111111111111111113") + const liveTxn = createRawTransaction("661111111111111111111114") + const model = createTransactionModel({ + replay: [replayedThenEvictedTxn, replayedRecentTxn], + changes: [replayedRecentTxn, liveTxn], + }) + const streamSettledTransactions = createStreamSettledTransactions({ + transactionModel: model.transactionModel, + maxReplayDedupeCacheSize: 1, + }) + + const stream = streamSettledTransactions({ + afterTransactionId: "661111111111111111111111" as LedgerTransactionId, + }) + + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111112" }, + done: false, + }) + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111113" }, + done: false, + }) + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111114" }, + done: false, + }) + await stream.return(undefined) + }) + + it("starts live-only when no cursor is provided", async () => { + const liveTxn = createRawTransaction("661111111111111111111113") + const model = createTransactionModel({ changes: [liveTxn] }) + const streamSettledTransactions = createStreamSettledTransactions({ + transactionModel: model.transactionModel, + }) + + const stream = streamSettledTransactions() + + await expect(stream.next()).resolves.toMatchObject({ + value: { id: "661111111111111111111113" }, + done: false, + }) + await stream.return(undefined) + + expect(model.find).not.toHaveBeenCalled() + expect(model.watch).toHaveBeenCalledWith( + [ + { + $match: expect.not.objectContaining({ + "fullDocument._id": expect.anything(), + }), + }, + ], + { fullDocument: "updateLookup" }, + ) + }) + + it("surfaces change stream errors as ledger errors", async () => { + const model = createTransactionModel({ + changeError: new Error("change stream failed"), + }) + const streamSettledTransactions = createStreamSettledTransactions({ + transactionModel: model.transactionModel, + }) + + const stream = streamSettledTransactions() + const result = await stream.next() + + expect(result.value).toBeInstanceOf(UnknownLedgerError) + await stream.return(undefined) + expect(model.changeStream.close).toHaveBeenCalled() + }) +}) diff --git a/core/api/test/unit/services/transactions-stream/helpers.spec.ts b/core/api/test/unit/services/transactions-stream/helpers.spec.ts index 2035648714..597c9394b5 100644 --- a/core/api/test/unit/services/transactions-stream/helpers.spec.ts +++ b/core/api/test/unit/services/transactions-stream/helpers.spec.ts @@ -1,12 +1,27 @@ -import mongoose from "mongoose" +jest.mock("@/services/ledger/schema", () => ({ + TransactionMetadata: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/schema", () => ({ + WalletInvoice: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/wallets", () => ({ + WalletsRepository: jest.fn(), +})) import { + createAccountIdLoader, createAccountIdResolver, createPreimageLoader, createPreimageResolver, createTransactionStreamEventMapper, - parseWalletId, } from "@/services/transactions-stream/helpers" +import { TransactionMetadata } from "@/services/ledger/schema" import { TransactionsStreamSettlementVia, @@ -16,14 +31,8 @@ import { import { LedgerTransactionType } from "@/domain/ledger" import { WalletCurrency } from "@/domain/shared" -describe("parseWalletId", () => { - it("returns the wallet id for liabilities accounts", () => { - expect(parseWalletId("Liabilities:wallet-123")).toBe("wallet-123") - }) - - it("returns undefined for non-liabilities accounts", () => { - expect(parseWalletId("Assets:wallet-123")).toBeUndefined() - }) +afterEach(() => { + jest.clearAllMocks() }) describe("mapLedgerTransactionTypeToSettlementVia", () => { @@ -59,6 +68,29 @@ describe("createAccountIdResolver", () => { expect(loadAccountId).toHaveBeenCalledTimes(1) }) + + it("does not cache unresolved wallet lookups", async () => { + const loadAccountId = jest + .fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce("account-1") + const resolveAccountId = createAccountIdResolver({ loadAccountId }) + + await expect(resolveAccountId("wallet-1" as WalletId)).resolves.toBeUndefined() + await expect(resolveAccountId("wallet-1" as WalletId)).resolves.toBe("account-1") + + expect(loadAccountId).toHaveBeenCalledTimes(2) + }) +}) + +describe("createAccountIdLoader", () => { + it("propagates wallet repository errors", async () => { + const lookupError = new Error("wallet lookup failed") + const findWalletById = jest.fn().mockResolvedValue(lookupError) + const loadAccountId = createAccountIdLoader({ findWalletById }) + + await expect(loadAccountId("wallet-1" as WalletId)).rejects.toBe(lookupError) + }) }) describe("createPreimageLoader", () => { @@ -74,7 +106,10 @@ describe("createPreimageLoader", () => { }) await expect( - loadPreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + loadPreimage({ + transactionId: "tx-1" as LedgerTransactionId, + paymentHash: "hash-1" as PaymentHash, + }), ).resolves.toBe("revealed-preimage") expect(findWalletInvoiceById).not.toHaveBeenCalled() }) @@ -92,9 +127,14 @@ describe("createPreimageLoader", () => { }) await expect( - loadPreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + loadPreimage({ + transactionId: "tx-1" as LedgerTransactionId, + paymentHash: "hash-1" as PaymentHash, + }), ).resolves.toBe("invoice-secret") - await expect(loadPreimage({ transactionId: "tx-2" })).resolves.toBe("") + await expect( + loadPreimage({ transactionId: "tx-2" as LedgerTransactionId }), + ).resolves.toBe("") }) }) @@ -104,10 +144,16 @@ describe("createPreimageResolver", () => { const resolvePreimage = createPreimageResolver({ loadPreimage }) await expect( - resolvePreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + resolvePreimage({ + transactionId: "tx-1" as LedgerTransactionId, + paymentHash: "hash-1" as PaymentHash, + }), ).resolves.toBe("preimage-1") await expect( - resolvePreimage({ transactionId: "tx-1", paymentHash: "hash-1" }), + resolvePreimage({ + transactionId: "tx-1" as LedgerTransactionId, + paymentHash: "hash-1" as PaymentHash, + }), ).resolves.toBe("preimage-1") expect(loadPreimage).toHaveBeenCalledTimes(1) @@ -124,18 +170,23 @@ describe("createTransactionStreamEventMapper", () => { }) const ledgerTransaction = { - _id: new mongoose.Types.ObjectId("661111111111111111111111"), - accounts: "Liabilities:wallet-1", - hash: "payment-hash-1", + id: "661111111111111111111111" as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + paymentHash: "payment-hash-1" as PaymentHash, type: LedgerTransactionType.Payment, - pending: false, + debit: 321 as Satoshis, + credit: -321 as Satoshis, + pendingConfirmation: false, currency: WalletCurrency.Btc, - satsAmount: 321, - centsAmount: 654, - credit: -321, - datetime: new Date("2024-01-01T00:00:05Z"), + journalId: "journal-1" as LedgerJournalId, + satsAmount: 321 as Satoshis, + centsAmount: 654 as UsdCents, timestamp: new Date("2024-01-01T00:00:00Z"), - } + feeKnownInAdvance: false, + fee: undefined, + usd: undefined, + feeUsd: undefined, + } as LedgerTransaction const event = await mapTransactionStreamEvent(ledgerTransaction) @@ -151,7 +202,66 @@ describe("createTransactionStreamEventMapper", () => { type: TransactionsStreamTransactionType.Sent, settlementVia: TransactionsStreamSettlementVia.Lightning, pending: false, - timestamp: new Date("2024-01-01T00:00:05Z"), + timestamp: new Date("2024-01-01T00:00:00Z"), + }) + }) + + it("skips ledger transactions without a wallet id", async () => { + const { mapTransactionStreamEvent } = createTransactionStreamEventMapper() + + const ledgerTransaction = { + id: "661111111111111111111111" as LedgerTransactionId, + walletId: undefined, + type: LedgerTransactionType.Payment, + debit: 0 as Satoshis, + credit: 1 as Satoshis, + pendingConfirmation: false, + currency: WalletCurrency.Btc, + journalId: "journal-1" as LedgerJournalId, + timestamp: new Date("2024-01-01T00:00:00Z"), + feeKnownInAdvance: false, + fee: undefined, + usd: undefined, + feeUsd: undefined, + } as LedgerTransaction + + await expect(mapTransactionStreamEvent(ledgerTransaction)).resolves.toBeUndefined() + }) + + it("maps transactions with the default preimage resolver after ledger schema is loaded", async () => { + const findById = TransactionMetadata.findById as jest.Mock + findById.mockReturnValueOnce({ + lean: jest.fn().mockResolvedValue(null), + } as never) + + const resolveAccountId = jest.fn().mockResolvedValue("account-1") + const { mapTransactionStreamEvent } = createTransactionStreamEventMapper({ + resolveAccountId, + }) + + const ledgerTransaction = { + id: "661111111111111111111111" as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + type: LedgerTransactionType.Invoice, + debit: 0 as Satoshis, + credit: 100 as Satoshis, + pendingConfirmation: false, + currency: WalletCurrency.Btc, + journalId: "journal-1" as LedgerJournalId, + satsAmount: 100 as Satoshis, + centsAmount: 200 as UsdCents, + timestamp: new Date("2024-01-01T00:00:00Z"), + feeKnownInAdvance: false, + fee: undefined, + usd: undefined, + feeUsd: undefined, + } as LedgerTransaction + + await expect(mapTransactionStreamEvent(ledgerTransaction)).resolves.toMatchObject({ + ledgerTransactionId: "661111111111111111111111", + accountId: "account-1", + preimage: "", }) + expect(findById).toHaveBeenCalledWith("661111111111111111111111") }) }) diff --git a/core/api/test/unit/services/transactions-stream/index.spec.ts b/core/api/test/unit/services/transactions-stream/index.spec.ts index 8d505a0db8..4be0968ea4 100644 --- a/core/api/test/unit/services/transactions-stream/index.spec.ts +++ b/core/api/test/unit/services/transactions-stream/index.spec.ts @@ -1,52 +1,66 @@ -import mongoose from "mongoose" - -import { - TransactionsStreamService, - TransactionStreamQueries, -} from "@/services/transactions-stream" -import { TransactionStreamRecord } from "@/services/transactions-stream/helpers" +jest.mock("@/services/ledger", () => ({ + LedgerService: jest.fn(() => ({ + streamSettledTransactions: jest.fn(), + })), +})) + +jest.mock("@/services/ledger/schema", () => ({ + TransactionMetadata: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/schema", () => ({ + WalletInvoice: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/wallets", () => ({ + WalletsRepository: jest.fn(), +})) + +import { TransactionsStreamService } from "@/services/transactions-stream" import { TransactionsStreamSettlementVia, TransactionsStreamTransactionType, } from "@/domain/transactions-stream" import { LedgerTransactionType } from "@/domain/ledger" +import { UnknownLedgerError } from "@/domain/ledger/errors" import { WalletCurrency } from "@/domain/shared" -jest.useFakeTimers() - -afterAll(() => { - jest.useRealTimers() -}) - afterEach(() => { jest.clearAllMocks() - jest.clearAllTimers() }) const flushMicrotasks = async () => { - await Promise.resolve() - await Promise.resolve() + await new Promise((resolve) => setImmediate(resolve)) } const createLedgerTransaction = ( id: string, - overrides: Partial = {}, -): TransactionStreamRecord => + overrides: Partial> = {}, +): LedgerTransaction => ({ - _id: new mongoose.Types.ObjectId(id), - accounts: "Liabilities:wallet-1", - hash: "payment-hash", + id: id as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + paymentHash: "payment-hash" as PaymentHash, type: LedgerTransactionType.Invoice, - pending: false, + debit: 0 as Satoshis, + credit: 100 as Satoshis, + pendingConfirmation: false, currency: WalletCurrency.Btc, - satsAmount: 100, - centsAmount: 200, - credit: 100, - datetime: new Date("2024-01-01T00:00:00Z"), + journalId: "journal-1" as LedgerJournalId, + satsAmount: 100 as Satoshis, + centsAmount: 200 as UsdCents, timestamp: new Date("2024-01-01T00:00:00Z"), + feeKnownInAdvance: false, + fee: undefined, + usd: undefined, + feeUsd: undefined, ...overrides, - }) as TransactionStreamRecord + }) as LedgerTransaction const createEvent = (ledgerTransactionId: string): TransactionStreamEvent => ({ ledgerTransactionId: ledgerTransactionId as LedgerTransactionId, @@ -63,6 +77,12 @@ const createEvent = (ledgerTransactionId: string): TransactionStreamEvent => ({ timestamp: new Date("2024-01-01T00:00:00Z"), }) +async function* ledgerTransactionGenerator( + values: Array | LedgerError>, +) { + for (const value of values) yield value +} + const subscribe = ({ service, afterTransactionId, @@ -83,12 +103,11 @@ const subscribe = ({ describe("TransactionsStreamService", () => { it("returns an error for malformed cursors", () => { - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter: jest.fn(), - findLatestTransactionId: jest.fn(), + const ledgerService = { + streamSettledTransactions: jest.fn(), } const service = TransactionsStreamService({ - transactionQueries, + ledgerService, mapTransactionStreamEvent: jest.fn(), }) @@ -98,171 +117,177 @@ describe("TransactionsStreamService", () => { }) expect(subscription).toBeInstanceOf(Error) - expect(transactionQueries.listSettledTransactionsAfter).not.toHaveBeenCalled() + expect(ledgerService.streamSettledTransactions).not.toHaveBeenCalled() + }) + + it("returns an error for explicitly empty cursors", () => { + const ledgerService = { + streamSettledTransactions: jest.fn(), + } + const service = TransactionsStreamService({ + ledgerService, + mapTransactionStreamEvent: jest.fn(), + }) + + const { subscription } = subscribe({ + service, + afterTransactionId: "", + }) + + expect(subscription).toBeInstanceOf(Error) + expect(ledgerService.streamSettledTransactions).not.toHaveBeenCalled() }) - it("replays transactions when after_transaction_id is provided", async () => { + it("streams translated ledger transactions from the ledger service", async () => { const replayTxn = createLedgerTransaction("661111111111111111111112") - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter: jest.fn().mockResolvedValue([replayTxn]), - findLatestTransactionId: jest.fn(), + const liveTxn = createLedgerTransaction("661111111111111111111113") + const ledgerService = { + streamSettledTransactions: jest + .fn() + .mockReturnValue(ledgerTransactionGenerator([replayTxn, liveTxn])), } const mapTransactionStreamEvent = jest .fn() - .mockResolvedValue(createEvent(replayTxn._id.toString())) + .mockImplementation(async (txn: LedgerTransaction) => + createEvent(txn.id), + ) const service = TransactionsStreamService({ - transactionQueries, + ledgerService, mapTransactionStreamEvent, - pollIntervalMs: 200, }) - const { onTransaction } = subscribe({ + const { onTransaction, onError } = subscribe({ service, afterTransactionId: "661111111111111111111111", }) await flushMicrotasks() - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledWith({ + expect(ledgerService.streamSettledTransactions).toHaveBeenCalledTimes(1) + expect(ledgerService.streamSettledTransactions).toHaveBeenCalledWith({ afterTransactionId: "661111111111111111111111", - limit: 100, + signal: expect.any(AbortSignal), }) - expect(onTransaction).toHaveBeenCalledTimes(1) - expect(onTransaction).toHaveBeenCalledWith(createEvent("661111111111111111111112")) - expect(transactionQueries.findLatestTransactionId).not.toHaveBeenCalled() + expect(mapTransactionStreamEvent).toHaveBeenNthCalledWith(1, replayTxn) + expect(mapTransactionStreamEvent).toHaveBeenNthCalledWith(2, liveTxn) + expect(onTransaction).toHaveBeenNthCalledWith( + 1, + createEvent("661111111111111111111112"), + ) + expect(onTransaction).toHaveBeenNthCalledWith( + 2, + createEvent("661111111111111111111113"), + ) + expect(onError).not.toHaveBeenCalled() }) - it("starts from the current tip when no cursor is provided", async () => { - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter: jest.fn().mockResolvedValue([]), - findLatestTransactionId: jest + it("starts at the ledger service live stream when no cursor is provided", async () => { + const ledgerService = { + streamSettledTransactions: jest .fn() - .mockResolvedValue("661111111111111111111111" as LedgerTransactionId), + .mockReturnValue(ledgerTransactionGenerator([])), } - const mapTransactionStreamEvent = jest.fn() const service = TransactionsStreamService({ - transactionQueries, - mapTransactionStreamEvent, - pollIntervalMs: 200, + ledgerService, + mapTransactionStreamEvent: jest.fn(), }) const { onTransaction } = subscribe({ service }) await flushMicrotasks() - expect(transactionQueries.listSettledTransactionsAfter).not.toHaveBeenCalled() - expect(mapTransactionStreamEvent).not.toHaveBeenCalled() - expect(onTransaction).not.toHaveBeenCalled() - - await jest.advanceTimersByTimeAsync(200) - - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledWith({ - afterTransactionId: "661111111111111111111111", - limit: 100, + expect(ledgerService.streamSettledTransactions).toHaveBeenCalledWith({ + afterTransactionId: undefined, + signal: expect.any(AbortSignal), }) + expect(onTransaction).not.toHaveBeenCalled() }) - it("polls new transactions after replay", async () => { - const replayTxn = createLedgerTransaction("661111111111111111111112") - const liveTxn = createLedgerTransaction("661111111111111111111113") - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter: jest + it("does not emit when the mapper skips a ledger transaction", async () => { + const ledgerTransaction = createLedgerTransaction("661111111111111111111112") + const ledgerService = { + streamSettledTransactions: jest .fn() - .mockResolvedValueOnce([replayTxn]) - .mockResolvedValueOnce([liveTxn]), - findLatestTransactionId: jest.fn(), + .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), } - const mapTransactionStreamEvent = jest - .fn() - .mockImplementation(async (txn: TransactionStreamRecord) => - createEvent(txn._id.toString()), - ) const service = TransactionsStreamService({ - transactionQueries, - mapTransactionStreamEvent, - pollIntervalMs: 200, + ledgerService, + mapTransactionStreamEvent: jest.fn().mockResolvedValue(undefined), }) - const { onTransaction } = subscribe({ - service, - afterTransactionId: "661111111111111111111111", - }) + const { onTransaction, onError } = subscribe({ service }) await flushMicrotasks() - await jest.advanceTimersByTimeAsync(200) - expect(onTransaction).toHaveBeenCalledTimes(2) - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenNthCalledWith(2, { - afterTransactionId: "661111111111111111111112", - limit: 100, - }) - expect(onTransaction).toHaveBeenNthCalledWith( - 2, - createEvent("661111111111111111111113"), - ) + expect(onTransaction).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() }) - it("does not overlap polling cycles", async () => { - const listSettledTransactionsAfter = jest.fn< - Promise, - [{ afterTransactionId: LedgerTransactionId; limit: number }] - >() - const findLatestTransactionId = jest - .fn, []>() - .mockResolvedValue("661111111111111111111111" as LedgerTransactionId) - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter, - findLatestTransactionId, + it("surfaces ledger errors from the stream", async () => { + const ledgerError = new UnknownLedgerError("stream failed") + const ledgerService = { + streamSettledTransactions: jest + .fn() + .mockReturnValue(ledgerTransactionGenerator([ledgerError])), } - let resolveFirstPoll: ((value: TransactionStreamRecord[]) => void) | undefined - const firstPoll = new Promise((resolve) => { - resolveFirstPoll = resolve - }) - listSettledTransactionsAfter.mockImplementationOnce(() => firstPoll) - listSettledTransactionsAfter.mockResolvedValueOnce([]) - const service = TransactionsStreamService({ - transactionQueries, + ledgerService, mapTransactionStreamEvent: jest.fn(), - pollIntervalMs: 200, + logger: { error: jest.fn() } as unknown as Logger, }) - subscribe({ service }) + const { onError } = subscribe({ service }) await flushMicrotasks() - await jest.advanceTimersByTimeAsync(600) - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) + expect(onError).toHaveBeenCalledWith(ledgerError) + }) - expect(resolveFirstPoll).toBeDefined() - resolveFirstPoll!([]) + it("surfaces mapper errors from the stream", async () => { + const ledgerTransaction = createLedgerTransaction("661111111111111111111112") + const mapperError = new Error("wallet lookup failed") + const ledgerService = { + streamSettledTransactions: jest + .fn() + .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), + } + const service = TransactionsStreamService({ + ledgerService, + mapTransactionStreamEvent: jest.fn().mockRejectedValue(mapperError), + logger: { error: jest.fn() } as unknown as Logger, + }) + + const { onTransaction, onError } = subscribe({ service }) await flushMicrotasks() - await jest.advanceTimersByTimeAsync(200) - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(2) + expect(onTransaction).not.toHaveBeenCalled() + expect(onError).toHaveBeenCalledWith(mapperError) }) - it("stops polling when the subscription is closed", async () => { - const transactionQueries: TransactionStreamQueries = { - listSettledTransactionsAfter: jest.fn().mockResolvedValue([]), - findLatestTransactionId: jest - .fn() - .mockResolvedValue("661111111111111111111111" as LedgerTransactionId), + it("aborts the ledger stream when the subscription is closed", async () => { + let streamSignal: AbortSignal | undefined + const ledgerService = { + streamSettledTransactions: jest.fn(({ signal }: StreamSettledTransactionsArgs) => { + streamSignal = signal + + return (async function* () { + await new Promise((resolve) => { + signal?.addEventListener("abort", () => resolve(), { once: true }) + }) + yield createLedgerTransaction("661111111111111111111112") + })() + }), } const service = TransactionsStreamService({ - transactionQueries, + ledgerService, mapTransactionStreamEvent: jest.fn(), - pollIntervalMs: 200, }) const { subscription } = subscribe({ service }) await flushMicrotasks() - await jest.advanceTimersByTimeAsync(200) - - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) expect(subscription).not.toBeInstanceOf(Error) if (subscription instanceof Error) return subscription.close() - await jest.advanceTimersByTimeAsync(400) + await flushMicrotasks() - expect(transactionQueries.listSettledTransactionsAfter).toHaveBeenCalledTimes(1) + expect(streamSignal?.aborted).toBe(true) }) }) From 04279344bcc1f28c2a097010407d87ac2117fa5b Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 30 Apr 2026 05:25:36 +0300 Subject: [PATCH 04/11] fix(api): harden transactions grpc stream delivery --- .../transactions-stream/grpc-server.ts | 25 ++- .../transactions-stream/grpc-server.spec.ts | 202 +++++++++++++++++- 2 files changed, 220 insertions(+), 7 deletions(-) diff --git a/core/api/src/services/transactions-stream/grpc-server.ts b/core/api/src/services/transactions-stream/grpc-server.ts index 969712164a..411a07fecb 100644 --- a/core/api/src/services/transactions-stream/grpc-server.ts +++ b/core/api/src/services/transactions-stream/grpc-server.ts @@ -12,6 +12,9 @@ import { baseLogger } from "@/services/logger" const logger = baseLogger.child({ module: "transactions-grpc-stream" }) +const requestAfterTransactionId = (request: SubscribeTransactionsRequest) => + request.hasAfterTransactionId() ? request.getAfterTransactionId() : undefined + const toServiceError = ({ code, message, @@ -49,11 +52,28 @@ export const TransactionsGrpcServer = ({ subscriptionRef.current?.close() } + const waitForDrainOrTerminalEvent = () => + new Promise((resolve) => { + const finish = () => { + call.removeListener("drain", finish) + call.removeListener("cancelled", finish) + call.removeListener("error", finish) + resolve() + } + + call.once("drain", finish) + call.once("cancelled", finish) + call.once("error", finish) + }) + const result = transactionsStreamService.subscribeToTransactions({ - afterTransactionId: call.request.getAfterTransactionId() || undefined, + afterTransactionId: requestAfterTransactionId(call.request), onTransaction: async (event) => { if (isClosed) return - call.write(transactionStreamEventToGrpcTransactionEvent(event)) + const canContinue = call.write( + transactionStreamEventToGrpcTransactionEvent(event), + ) + if (!canContinue && !isClosed) await waitForDrainOrTerminalEvent() }, onError: (err) => { serviceLogger.error({ err }, "Failed to stream transactions") @@ -76,7 +96,6 @@ export const TransactionsGrpcServer = ({ subscriptionRef.current = result call.on("cancelled", cleanup) - call.on("close", cleanup) call.on("error", cleanup) } diff --git a/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts b/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts index 450335b662..5e066efaba 100644 --- a/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts +++ b/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts @@ -1,10 +1,34 @@ import { EventEmitter } from "events" +jest.mock("@/services/ledger", () => ({ + LedgerService: jest.fn(() => ({ + streamSettledTransactions: jest.fn(), + })), +})) + +jest.mock("@/services/ledger/schema", () => ({ + TransactionMetadata: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/schema", () => ({ + WalletInvoice: { + findById: jest.fn(), + }, +})) + +jest.mock("@/services/mongoose/wallets", () => ({ + WalletsRepository: jest.fn(), +})) + import { status } from "@grpc/grpc-js" +import { TransactionsStreamService } from "@/services/transactions-stream" import { TransactionsGrpcServer } from "@/services/transactions-stream/grpc-server" import { SubscribeTransactionsRequest } from "@/services/transactions-stream/proto/transactions_pb" +import { LedgerTransactionType } from "@/domain/ledger" import { TransactionsStreamSettlementVia, TransactionsStreamTransactionType, @@ -18,8 +42,7 @@ type MockCall = EventEmitter & { } const flushMicrotasks = async () => { - await Promise.resolve() - await Promise.resolve() + await new Promise((resolve) => setImmediate(resolve)) } const createCall = (afterTransactionId?: string): MockCall => { @@ -28,7 +51,7 @@ const createCall = (afterTransactionId?: string): MockCall => { const call = new EventEmitter() as MockCall call.request = request - call.write = jest.fn() + call.write = jest.fn().mockReturnValue(true) call.destroy = jest.fn() return call @@ -49,6 +72,30 @@ const createEvent = (): TransactionStreamEvent => ({ timestamp: new Date("2024-01-01T00:00:05Z"), }) +const createLedgerTransaction = (id: string): LedgerTransaction => + ({ + id: id as LedgerTransactionId, + walletId: "wallet-1" as WalletId, + paymentHash: "payment-hash" as PaymentHash, + type: LedgerTransactionType.Invoice, + debit: 0 as Satoshis, + credit: 100 as Satoshis, + pendingConfirmation: false, + currency: WalletCurrency.Btc, + journalId: "journal-1" as LedgerJournalId, + satsAmount: 100 as Satoshis, + centsAmount: 200 as UsdCents, + timestamp: new Date("2024-01-01T00:00:05Z"), + feeKnownInAdvance: false, + fee: undefined, + usd: undefined, + feeUsd: undefined, + }) as LedgerTransaction + +async function* ledgerTransactionGenerator(values: LedgerTransaction[]) { + for (const value of values) yield value +} + describe("TransactionsGrpcServer", () => { it("returns INVALID_ARGUMENT for malformed cursors", () => { const transactionsStreamService = { @@ -70,7 +117,27 @@ describe("TransactionsGrpcServer", () => { }) }) - it("maps domain events to grpc messages and closes the subscription", async () => { + it("returns INVALID_ARGUMENT for explicitly empty cursors", () => { + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockReturnValue(new Error("invalid cursor")), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("") + + grpcServer.subscribeTransactions(call as never) + + expect(call.destroy).toHaveBeenCalledTimes(1) + expect(call.destroy.mock.calls[0][0].code).toBe(status.INVALID_ARGUMENT) + expect(transactionsStreamService.subscribeToTransactions).toHaveBeenCalledWith({ + afterTransactionId: "", + onTransaction: expect.any(Function), + onError: expect.any(Function), + }) + }) + + it("maps domain events to grpc messages", async () => { const close = jest.fn() const transactionsStreamService = { subscribeToTransactions: jest.fn().mockImplementation(({ onTransaction }) => { @@ -98,9 +165,136 @@ describe("TransactionsGrpcServer", () => { expect(call.write.mock.calls[0][0].getCentsAmount()).toBe(200) expect(call.write.mock.calls[0][0].getCurrency()).toBe(WalletCurrency.Btc) expect(call.write.mock.calls[0][0].getTimestamp()).toBe(1704067205) + expect(close).not.toHaveBeenCalled() + }) + it("replays transactions through grpc even if close fires during stream setup", async () => { + const ledgerTransaction = createLedgerTransaction("661111111111111111111111") + const ledgerService = { + streamSettledTransactions: jest + .fn() + .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), + } + const mapTransactionStreamEvent = jest.fn().mockResolvedValue(createEvent()) + const transactionsStreamService = TransactionsStreamService({ + ledgerService, + mapTransactionStreamEvent, + }) + const grpcServer = TransactionsGrpcServer({ transactionsStreamService }) + const call = createCall("000000000000000000000000") + + grpcServer.subscribeTransactions(call as never) + call.emit("close") + await flushMicrotasks() + + expect(ledgerService.streamSettledTransactions).toHaveBeenCalledWith({ + afterTransactionId: "000000000000000000000000", + signal: expect.any(AbortSignal), + }) + expect(mapTransactionStreamEvent).toHaveBeenCalledWith(ledgerTransaction) + expect(call.write).toHaveBeenCalledTimes(1) + }) + + it("does not close the subscription on grpc close", () => { + const close = jest.fn() + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockReturnValue({ close }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("661111111111111111111110") + + grpcServer.subscribeTransactions(call as never) call.emit("close") + expect(close).not.toHaveBeenCalled() + }) + + it("closes the subscription on client cancellation", () => { + const close = jest.fn() + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockReturnValue({ close }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("661111111111111111111110") + + grpcServer.subscribeTransactions(call as never) + call.emit("cancelled") + + expect(close).toHaveBeenCalledTimes(1) + }) + + it("closes the subscription on grpc call errors", () => { + const close = jest.fn() + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockReturnValue({ close }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("661111111111111111111110") + + grpcServer.subscribeTransactions(call as never) + call.emit("error", new Error("client stream failed")) + + expect(close).toHaveBeenCalledTimes(1) + }) + + it("destroys the grpc call on service stream errors", () => { + const close = jest.fn() + const streamError = new Error("change stream failed") + let onError: ((err: Error) => void) | undefined + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockImplementation(({ onError: handler }) => { + onError = handler + return { close } + }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + logger: { error: jest.fn() } as unknown as Logger, + }) + const call = createCall("661111111111111111111110") + + grpcServer.subscribeTransactions(call as never) + onError?.(streamError) + expect(close).toHaveBeenCalledTimes(1) + expect(call.destroy).toHaveBeenCalledWith(streamError) + }) + + it("waits for grpc drain when writes apply backpressure", async () => { + let onTransactionResult: Promise | undefined + const transactionsStreamService = { + subscribeToTransactions: jest.fn().mockImplementation(({ onTransaction }) => { + onTransactionResult = onTransaction(createEvent()) + return { close: jest.fn() } + }), + } + const grpcServer = TransactionsGrpcServer({ + transactionsStreamService: transactionsStreamService as never, + }) + const call = createCall("661111111111111111111110") + call.write.mockReturnValueOnce(false) + + grpcServer.subscribeTransactions(call as never) + await flushMicrotasks() + + let didFinish = false + onTransactionResult?.then(() => { + didFinish = true + }) + await flushMicrotasks() + + expect(call.write).toHaveBeenCalledTimes(1) + expect(didFinish).toBe(false) + + call.emit("drain") + await onTransactionResult + + expect(didFinish).toBe(true) }) }) From 58bfe4bfe041018987d5027a7fec6e8c2a194f46 Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 30 Apr 2026 05:26:22 +0300 Subject: [PATCH 05/11] test(api): wire transactions grpc stream into e2e --- bats/ci_run.sh | 2 +- bats/ci_setup_suite.bash | 6 ++ bats/core/api/transactions-grpc-stream.bats | 70 +++++++++++++++++++ bats/helpers/_common.bash | 3 +- bats/helpers/transactions-grpc-stream.bash | 25 +++++++ .../transactions-grpc-stream-server.ts | 20 +++--- dev/Tiltfile | 25 +++++++ dev/docker-compose.deps.yml | 10 +++ flake.nix | 11 ++- 9 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 bats/core/api/transactions-grpc-stream.bats create mode 100644 bats/helpers/transactions-grpc-stream.bash diff --git a/bats/ci_run.sh b/bats/ci_run.sh index d218754a00..70d12d1f3b 100755 --- a/bats/ci_run.sh +++ b/bats/ci_run.sh @@ -7,7 +7,7 @@ echo "Running rust builds..." buck2 build //core/api-keys:api-keys //core/notifications:notifications echo "Running api builds..." -buck2 build //core/api:api //core/api-ws-server:api-ws-server //core/api-trigger:api-trigger //core/api-exporter:api-exporter --verbose 4 +buck2 build //core/api:api //core/api-ws-server:api-ws-server //core/api-trigger:api-trigger //core/api-exporter:api-exporter //core/api-transactions-grpc-stream:api-transactions-grpc-stream --verbose 4 # echo "Running apps builds..." # buck2 build //apps/dashboard:dashboard //apps/consent:consent //apps/pay:pay-ci //apps/admin-panel:admin-panel //apps/map:map //apps/voucher:voucher --verbose 4 diff --git a/bats/ci_setup_suite.bash b/bats/ci_setup_suite.bash index 65a49c8a13..77a5f098a0 100644 --- a/bats/ci_setup_suite.bash +++ b/bats/ci_setup_suite.bash @@ -2,6 +2,7 @@ export REPO_ROOT=$(git rev-parse --show-toplevel) source "${REPO_ROOT}/bats/helpers/_common.bash" +source "${REPO_ROOT}/bats/helpers/transactions-grpc-stream.bash" TILT_PID_FILE="${BATS_ROOT_DIR}/.tilt_pid" @@ -11,6 +12,7 @@ setup_suite() { await_notifications_is_up await_api_keys_is_up await_api_is_up + await_transactions_grpc_stream_is_up await_pay_is_up } @@ -55,3 +57,7 @@ await_notifications_is_up() { retry 360 2 notifications_is_up } + +await_transactions_grpc_stream_is_up() { + retry 360 2 transactions_grpc_stream_is_up +} diff --git a/bats/core/api/transactions-grpc-stream.bats b/bats/core/api/transactions-grpc-stream.bats new file mode 100644 index 0000000000..598573d549 --- /dev/null +++ b/bats/core/api/transactions-grpc-stream.bats @@ -0,0 +1,70 @@ +#!/usr/bin/env bats + +load "../../helpers/_common.bash" +load "../../helpers/onchain.bash" +load "../../helpers/transactions-grpc-stream.bash" +load "../../helpers/user.bash" + +ALICE='alice' + +setup_file() { + clear_cache + create_user "$ALICE" +} + +create_settled_onchain_receive_and_get_transaction_id() { + local token_name=$1 + local wallet_id_name="$token_name.btc_wallet_id" + + variables=$( + jq -n \ + --arg wallet_id "$(read_value $wallet_id_name)" \ + '{input: {walletId: $wallet_id}}' + ) + exec_graphql "$token_name" 'on-chain-address-create' "$variables" >&2 + address="$(graphql_output '.data.onChainAddressCreate.address')" + [[ "${address}" != "null" ]] || exit 1 + + bitcoin_cli sendtoaddress "$address" "0.001" >&2 + bitcoin_cli -generate 2 >&2 + retry 30 1 check_for_onchain_initiated_settled "$token_name" "$address" >&2 + + exec_graphql "$token_name" 'transactions' '{"first": 20}' >&2 + transaction_id="$(get_from_transaction_by_address "$address" '.id')" + [[ -n "${transaction_id}" ]] || exit 1 + [[ "${transaction_id}" != "null" ]] || exit 1 + + echo "$transaction_id" +} + +@test "transactions-grpc-stream: invalid cursor returns invalid argument" { + transactions_grpc_stream_request '{"after_transaction_id":"not-an-object-id"}' + + [[ "$status" -ne 0 ]] || exit 1 + echo "$output" | grep -F "InvalidArgument" || exit 1 + echo "$output" | grep -F "after_transaction_id must be a valid Mongo ObjectId" || exit 1 +} + +@test "transactions-grpc-stream: empty cursor returns invalid argument" { + transactions_grpc_stream_request '{"after_transaction_id":""}' + + [[ "$status" -ne 0 ]] || exit 1 + echo "$output" | grep -F "InvalidArgument" || exit 1 + echo "$output" | grep -F "after_transaction_id must be a valid Mongo ObjectId" || exit 1 +} + +@test "transactions-grpc-stream: replays transactions after cursor" { + first_transaction_id="$(create_settled_onchain_receive_and_get_transaction_id "$ALICE")" + second_transaction_id="$(create_settled_onchain_receive_and_get_transaction_id "$ALICE")" + + data=$( + jq -n \ + --arg after_transaction_id "$first_transaction_id" \ + '{after_transaction_id: $after_transaction_id}' + ) + transactions_grpc_stream_request "$data" -max-time 3 + + echo "$output" | grep -F "\"ledgerTransactionId\": \"$second_transaction_id\"" || exit 1 + echo "$output" | grep -F "\"walletId\": \"$(read_value "$ALICE.btc_wallet_id")\"" || exit 1 + echo "$output" | grep -F "\"accountId\": \"$(read_value "$ALICE.account_id")\"" || exit 1 +} diff --git a/bats/helpers/_common.bash b/bats/helpers/_common.bash index dd70d09c27..082007aef1 100644 --- a/bats/helpers/_common.bash +++ b/bats/helpers/_common.bash @@ -220,6 +220,7 @@ grpcurl_request() { local address="$3" local service_method="$4" local data="${5:-""}" + shift 5 echo "gRPCurl request - import-path ${import_path} - proto: ${proto_file} - address: ${address} - service/method: ${service_method} - data: ${data}" @@ -228,7 +229,7 @@ grpcurl_request() { run_cmd="run" fi - cmd=(${run_cmd} grpcurl -plaintext -import-path ${import_path} -proto "${proto_file}") + cmd=(${run_cmd} grpcurl -plaintext "$@" -import-path ${import_path} -proto "${proto_file}") if [[ -n "$data" ]]; then cmd+=(-d "${data}") diff --git a/bats/helpers/transactions-grpc-stream.bash b/bats/helpers/transactions-grpc-stream.bash new file mode 100644 index 0000000000..41d8f1322a --- /dev/null +++ b/bats/helpers/transactions-grpc-stream.bash @@ -0,0 +1,25 @@ +CURRENT_FILE=${BASH_SOURCE:-bats/helpers/.} +source "$(dirname "$CURRENT_FILE")/_common.bash" + +export TRANSACTIONS_GRPC_STREAM_ADDRESS="${TRANSACTIONS_GRPC_STREAM_ADDRESS:-localhost:50053}" +export TRANSACTIONS_GRPC_STREAM_HEALTH_URL="${TRANSACTIONS_GRPC_STREAM_HEALTH_URL:-http://localhost:8889/healthz}" +export TRANSACTIONS_GRPC_STREAM_PROTO_IMPORT_PATH="${REPO_ROOT}/core/api/src/services/transactions-stream/proto" +export TRANSACTIONS_GRPC_STREAM_PROTO_FILE="transactions.proto" +export TRANSACTIONS_GRPC_STREAM_SERVICE_METHOD="services.transactions.v1.TransactionsStream/SubscribeTransactions" + +transactions_grpc_stream_request() { + local data="${1:-""}" + shift || true + + grpcurl_request \ + "${TRANSACTIONS_GRPC_STREAM_PROTO_IMPORT_PATH}" \ + "${TRANSACTIONS_GRPC_STREAM_PROTO_FILE}" \ + "${TRANSACTIONS_GRPC_STREAM_ADDRESS}" \ + "${TRANSACTIONS_GRPC_STREAM_SERVICE_METHOD}" \ + "${data}" \ + "$@" +} + +transactions_grpc_stream_is_up() { + curl -fsS "${TRANSACTIONS_GRPC_STREAM_HEALTH_URL}" > /dev/null +} diff --git a/core/api/src/servers/transactions-grpc-stream-server.ts b/core/api/src/servers/transactions-grpc-stream-server.ts index 3fa9b34d30..ca17720aa3 100644 --- a/core/api/src/servers/transactions-grpc-stream-server.ts +++ b/core/api/src/servers/transactions-grpc-stream-server.ts @@ -16,7 +16,7 @@ import { TransactionsStreamService } from "@/services/transactions-stream/proto/ const logger = baseLogger.child({ module: "transactions-grpc-stream-server" }) -const startHealthServer = () => { +const startHealthServer = async () => { const app = express() app.get( @@ -29,11 +29,15 @@ const startHealthServer = () => { }), ) - app.listen(TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, () => { - logger.info( - { port: TRANSACTIONS_GRPC_STREAM_HEALTH_PORT }, - "Transactions gRPC stream health server listening", - ) + await new Promise((resolve, reject) => { + const healthServer = app.listen(TRANSACTIONS_GRPC_STREAM_HEALTH_PORT, () => { + logger.info( + { port: TRANSACTIONS_GRPC_STREAM_HEALTH_PORT }, + "Transactions gRPC stream health server listening", + ) + resolve() + }) + healthServer.once("error", reject) }) } @@ -54,12 +58,12 @@ const startGrpcServer = async () => { } const main = async () => { - startHealthServer() - await setupMongoConnection({ syncIndexes: false }) await startGrpcServer() + await startHealthServer() } main().catch((err) => { logger.error({ err }, "Transactions gRPC stream server failed") + process.exit(1) }) diff --git a/dev/Tiltfile b/dev/Tiltfile index 575ca65771..e6d7cd1074 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -613,6 +613,31 @@ local_resource( ] ) +api_transactions_grpc_stream_target = "//core/api-transactions-grpc-stream:dev" +if is_ci: + api_transactions_grpc_stream_target = ( + "//core/api-transactions-grpc-stream:api-transactions-grpc-stream" + ) +local_resource( + "api-transactions-grpc-stream", + labels = ["core"], + cmd = "buck2 build {}".format(api_transactions_grpc_stream_target), + serve_cmd = "buck2 run {}".format(api_transactions_grpc_stream_target), + serve_env = core_serve_env, + allow_parallel = True, + readiness_probe = probe( + period_secs = 5, + http_get = http_get_action( + path = "healthz", + port = 8889, + ), + ), + deps = _buck2_dep_inputs(api_transactions_grpc_stream_target), + resource_deps = [ + "mongodb", + ] +) + notifications_target = "//core/notifications:notifications" local_resource( "notifications", diff --git a/dev/docker-compose.deps.yml b/dev/docker-compose.deps.yml index d7d8dbb6f2..6ebf8f2e73 100644 --- a/dev/docker-compose.deps.yml +++ b/dev/docker-compose.deps.yml @@ -173,10 +173,20 @@ services: - REDIS_DISABLE_COMMANDS=FLUSHDB,FLUSHALL mongodb: image: mongo:7.0.6 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] ports: - "27017:27017" environment: - MONGO_INITDB_DATABASE=galoy + healthcheck: + test: + [ + "CMD-SHELL", + "mongosh --quiet --eval 'try { rs.status().ok } catch (e) { rs.initiate({ _id: \"rs0\", members: [{ _id: 0, host: \"localhost:27017\" }] }).ok }' | grep 1", + ] + interval: 5s + timeout: 30s + retries: 30 lnd1: image: lightninglabs/lnd:v0.19.3-beta ports: diff --git a/flake.nix b/flake.nix index 1d63134099..b97944bced 100644 --- a/flake.nix +++ b/flake.nix @@ -42,7 +42,14 @@ (import rust-overlay) ]; pkgs = import nixpkgs {inherit overlays system;}; - bufPkg = if pkgs.stdenv.isDarwin then dockerPkgs.buf else pkgs.buf; + bufPkg = + if pkgs.stdenv.isDarwin + then dockerPkgs.buf + else pkgs.buf; + grpcurlPkg = + if pkgs.stdenv.isDarwin + then dockerPkgs.grpcurl + else pkgs.grpcurl; rustVersion = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; rust-toolchain = rustVersion.override { extensions = ["rust-analyzer" "rust-src"]; @@ -84,7 +91,7 @@ cargo-watch reindeer gitMinimal - grpcurl + grpcurlPkg bufPkg netcat ] From 52154b99ca75f7995d60c7d70627fe6238633bbc Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 30 Apr 2026 06:20:28 +0300 Subject: [PATCH 06/11] fix(deps): bump protobufjs override --- package.json | 2 +- pnpm-lock.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 1d79790f80..594af9e2b3 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "resolutions": { "@types/ws": "^8.5.12", "basic-ftp": "^5.2.0", - "protobufjs": "7.2.5", + "protobufjs": "7.5.5", "http-cache-semantics": "4.1.1", "elliptic": "^6.6.1", "form-data": "^4.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7711c71dd..eddbff9943 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,7 +7,7 @@ settings: overrides: '@types/ws': ^8.5.12 basic-ftp: ^5.2.0 - protobufjs: 7.2.5 + protobufjs: 7.5.5 http-cache-semantics: 4.1.1 elliptic: ^6.6.1 form-data: ^4.0.4 @@ -5630,7 +5630,7 @@ packages: fast-deep-equal: 3.1.3 functional-red-black-tree: 1.0.1 google-gax: 4.6.1 - protobufjs: 7.2.5 + protobufjs: 7.5.5 transitivePeerDependencies: - encoding - supports-color @@ -7431,7 +7431,7 @@ packages: dependencies: lodash.camelcase: 4.3.0 long: 5.3.2 - protobufjs: 7.2.5 + protobufjs: 7.5.5 yargs: 17.7.2 dev: false @@ -8757,7 +8757,7 @@ packages: '@opentelemetry/sdk-logs': 0.53.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 1.26.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) - protobufjs: 7.2.5 + protobufjs: 7.5.5 dev: false /@opentelemetry/propagator-b3@1.26.0(@opentelemetry/api@1.9.0): @@ -21608,7 +21608,7 @@ packages: node-fetch: 2.7.0 object-hash: 3.0.0 proto3-json-serializer: 2.0.2 - protobufjs: 7.2.5 + protobufjs: 7.5.5 retry-request: 7.0.2 uuid: 9.0.1 transitivePeerDependencies: @@ -27132,12 +27132,12 @@ packages: engines: {node: '>=14.0.0'} requiresBuild: true dependencies: - protobufjs: 7.2.5 + protobufjs: 7.5.5 dev: false optional: true - /protobufjs@7.2.5: - resolution: {integrity: sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==} + /protobufjs@7.5.5: + resolution: {integrity: sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==} engines: {node: '>=12.0.0'} requiresBuild: true dependencies: From 02237819d96bdaaf10bfe1cdc84fe260a326748b Mon Sep 17 00:00:00 2001 From: basarrcan Date: Wed, 6 May 2026 04:50:42 +0300 Subject: [PATCH 07/11] refactor: move transaction stream server to servers layer --- bats/helpers/transactions-grpc-stream.bash | 2 +- core/api/package.json | 4 +- .../transactions-stream/helpers.ts | 84 ++++------------- .../transactions-stream/index.ts | 22 +---- .../app/transactions-stream/index.types.d.ts | 89 +++++++++++++++++++ .../transactions-grpc-stream-server.ts | 6 +- .../transactions-grpc-stream}/convert.ts | 0 .../transactions-grpc-stream}/grpc-server.ts | 22 ++--- .../transactions-grpc-stream/index.types.d.ts | 16 ++++ .../proto/buf.gen.yaml | 0 .../proto/transactions.proto | 0 .../proto/transactions_grpc_pb.d.ts | 0 .../proto/transactions_grpc_pb.js | 0 .../proto/transactions_pb.d.ts | 0 .../proto/transactions_pb.js | 0 .../transactions-stream/helpers.spec.ts | 2 +- .../transactions-stream/index.spec.ts | 22 ++--- .../grpc-server.spec.ts | 48 +++++----- 18 files changed, 173 insertions(+), 144 deletions(-) rename core/api/src/{services => app}/transactions-stream/helpers.ts (66%) rename core/api/src/{services => app}/transactions-stream/index.ts (80%) create mode 100644 core/api/src/app/transactions-stream/index.types.d.ts rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/convert.ts (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/grpc-server.ts (82%) create mode 100644 core/api/src/servers/transactions-grpc-stream/index.types.d.ts rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/buf.gen.yaml (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/transactions.proto (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/transactions_grpc_pb.d.ts (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/transactions_grpc_pb.js (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/transactions_pb.d.ts (100%) rename core/api/src/{services/transactions-stream => servers/transactions-grpc-stream}/proto/transactions_pb.js (100%) rename core/api/test/unit/{services => app}/transactions-stream/helpers.spec.ts (99%) rename core/api/test/unit/{services => app}/transactions-stream/index.spec.ts (93%) rename core/api/test/unit/{services/transactions-stream => servers/transactions-grpc-stream}/grpc-server.spec.ts (85%) diff --git a/bats/helpers/transactions-grpc-stream.bash b/bats/helpers/transactions-grpc-stream.bash index 41d8f1322a..c5018b1542 100644 --- a/bats/helpers/transactions-grpc-stream.bash +++ b/bats/helpers/transactions-grpc-stream.bash @@ -3,7 +3,7 @@ source "$(dirname "$CURRENT_FILE")/_common.bash" export TRANSACTIONS_GRPC_STREAM_ADDRESS="${TRANSACTIONS_GRPC_STREAM_ADDRESS:-localhost:50053}" export TRANSACTIONS_GRPC_STREAM_HEALTH_URL="${TRANSACTIONS_GRPC_STREAM_HEALTH_URL:-http://localhost:8889/healthz}" -export TRANSACTIONS_GRPC_STREAM_PROTO_IMPORT_PATH="${REPO_ROOT}/core/api/src/services/transactions-stream/proto" +export TRANSACTIONS_GRPC_STREAM_PROTO_IMPORT_PATH="${REPO_ROOT}/core/api/src/servers/transactions-grpc-stream/proto" export TRANSACTIONS_GRPC_STREAM_PROTO_FILE="transactions.proto" export TRANSACTIONS_GRPC_STREAM_SERVICE_METHOD="services.transactions.v1.TransactionsStream/SubscribeTransactions" diff --git a/core/api/package.json b/core/api/package.json index 8065e2c3e3..ceb5ccdafc 100644 --- a/core/api/package.json +++ b/core/api/package.json @@ -5,7 +5,7 @@ "eslint-check": "eslint src test --ext .ts", "eslint-fix": "eslint src test --ext .ts --fix", "circular-deps-check": "madge --circular --extensions ts src", - "build": "tsc -p tsconfig-build.json && cp -R src/services/price/protos dist/services/price/ && cp -R src/services/dealer-price/proto dist/services/dealer-price/ && cp -R src/services/bria/proto dist/services/bria/ && cp -R src/services/notifications/proto dist/services/notifications/ && cp -R src/services/api-keys/proto dist/services/api-keys/ && cp -R src/services/transactions-stream/proto dist/services/transactions-stream/ && tscpaths --silent -p tsconfig.json -s ./src -o ./dist", + "build": "tsc -p tsconfig-build.json && cp -R src/services/price/protos dist/services/price/ && cp -R src/services/dealer-price/proto dist/services/dealer-price/ && cp -R src/services/bria/proto dist/services/bria/ && cp -R src/services/notifications/proto dist/services/notifications/ && cp -R src/services/api-keys/proto dist/services/api-keys/ && cp -R src/servers/transactions-grpc-stream/proto dist/servers/transactions-grpc-stream/ && tscpaths --silent -p tsconfig.json -s ./src -o ./dist", "trigger": "pnpm run build && node dist/servers/trigger.js | pino-pretty -c -l", "ws": "pnpm run build && node dist/servers/ws-server.js | pino-pretty -c -l", "watch": "nodemon -V -e ts,graphql -w ./src -x pnpm run start", @@ -35,7 +35,7 @@ "mongodb-migrate": "pnpm run migrate:status && pnpm run migrate:up && pnpm run migrate:status", "codegen:notifications": "cd ./src/services/notifications/proto && buf generate", "codegen:api-keys": "cd ./src/services/api-keys/proto && buf generate", - "codegen:transactions-stream": "cd ./src/services/transactions-stream/proto && buf generate" + "codegen:transactions-stream": "cd ./src/servers/transactions-grpc-stream/proto && buf generate" }, "engines": { "node": "20" diff --git a/core/api/src/services/transactions-stream/helpers.ts b/core/api/src/app/transactions-stream/helpers.ts similarity index 66% rename from core/api/src/services/transactions-stream/helpers.ts rename to core/api/src/app/transactions-stream/helpers.ts index 4561ba350a..6f21541c4e 100644 --- a/core/api/src/services/transactions-stream/helpers.ts +++ b/core/api/src/app/transactions-stream/helpers.ts @@ -10,40 +10,10 @@ import { WalletsRepository } from "@/services/mongoose/wallets" const PREIMAGE_CACHE_TTL_MS = 5 * 60 * 1000 const PREIMAGE_CACHE_MAX_SIZE = 10_000 -type TimestampedValue = { - value: T - expiresAt: number -} - -type PreimageLoaderArgs = { - transactionId: LedgerTransactionId - paymentHash?: PaymentHash -} - -export type AccountIdLoader = (walletId: WalletId) => Promise -export type PreimageLoader = (args: PreimageLoaderArgs) => Promise -export type AccountIdResolver = (walletId: WalletId) => Promise -export type PreimageResolver = (args: PreimageLoaderArgs) => Promise - -type FindWalletById = ( - walletId: WalletId, -) => Promise<{ accountId: AccountId } | Error | undefined> -type FindTransactionMetadataById = ( - transactionId: LedgerTransactionId, -) => Promise | null | undefined> -type FindWalletInvoiceById = ( - paymentHash: string, -) => Promise | null | undefined> - class ExpiringCache { - private readonly values = new Map>() + private readonly values = new Map>() - constructor( - private readonly options: { - ttlMs: number - maxSize: number - }, - ) {} + constructor(private readonly options: TransactionsStreamExpiringCacheOptions) {} get(key: K): V | undefined { const cached = this.values.get(key) @@ -70,20 +40,21 @@ class ExpiringCache { } } -const defaultFindWalletById: FindWalletById = async (walletId) => { +const defaultFindWalletById: TransactionsStreamFindWalletById = async (walletId) => { return WalletsRepository().findById(walletId) } -const defaultFindTransactionMetadataById: FindTransactionMetadataById = async ( - transactionId, -) => { - return (await TransactionMetadata.findById(transactionId).lean()) as Pick< - TransactionMetadataRecord, - "revealedPreImage" - > | null -} +const defaultFindTransactionMetadataById: TransactionsStreamFindTransactionMetadataById = + async (transactionId) => { + return (await TransactionMetadata.findById(transactionId).lean()) as Pick< + TransactionMetadataRecord, + "revealedPreImage" + > | null + } -const defaultFindWalletInvoiceById: FindWalletInvoiceById = async (paymentHash) => { +const defaultFindWalletInvoiceById: TransactionsStreamFindWalletInvoiceById = async ( + paymentHash, +) => { return (await WalletInvoice.findById(paymentHash).lean()) as Pick< WalletInvoiceRecord, "secret" @@ -92,9 +63,7 @@ const defaultFindWalletInvoiceById: FindWalletInvoiceById = async (paymentHash) export const createAccountIdLoader = ({ findWalletById = defaultFindWalletById, -}: { - findWalletById?: FindWalletById -} = {}): AccountIdLoader => { +}: TransactionsStreamAccountIdLoaderConfig = {}): TransactionsStreamAccountIdLoader => { return async (walletId) => { const wallet = await findWalletById(walletId) if (wallet instanceof Error) throw wallet @@ -107,10 +76,7 @@ export const createAccountIdLoader = ({ export const createPreimageLoader = ({ findTransactionMetadataById = defaultFindTransactionMetadataById, findWalletInvoiceById = defaultFindWalletInvoiceById, -}: { - findTransactionMetadataById?: FindTransactionMetadataById - findWalletInvoiceById?: FindWalletInvoiceById -} = {}): PreimageLoader => { +}: TransactionsStreamPreimageLoaderConfig = {}): TransactionsStreamPreimageLoader => { return async ({ transactionId, paymentHash }) => { const txMetadata = await findTransactionMetadataById(transactionId) if (txMetadata?.revealedPreImage) return txMetadata.revealedPreImage @@ -124,10 +90,7 @@ export const createPreimageLoader = ({ export const createAccountIdResolver = ({ walletToAccountCache = new Map(), loadAccountId = createAccountIdLoader(), -}: { - walletToAccountCache?: Map - loadAccountId?: AccountIdLoader -} = {}): AccountIdResolver => { +}: TransactionsStreamAccountIdResolverConfig = {}): TransactionsStreamAccountIdResolver => { return async (walletId: WalletId) => { if (walletToAccountCache.has(walletId)) { return walletToAccountCache.get(walletId) @@ -146,14 +109,8 @@ export const createPreimageResolver = ({ maxSize: PREIMAGE_CACHE_MAX_SIZE, }), loadPreimage = createPreimageLoader(), -}: { - preimageCache?: { - get: (key: string) => string | undefined - set: (key: string, value: string) => void - } - loadPreimage?: PreimageLoader -} = {}): PreimageResolver => { - return async ({ transactionId, paymentHash }: PreimageLoaderArgs) => { +}: TransactionsStreamPreimageResolverConfig = {}): TransactionsStreamPreimageResolver => { + return async ({ transactionId, paymentHash }: TransactionsStreamPreimageLoaderArgs) => { const cached = preimageCache.get(transactionId) if (cached !== undefined) return cached @@ -166,10 +123,7 @@ export const createPreimageResolver = ({ export const createTransactionStreamEventMapper = ({ resolveAccountId = createAccountIdResolver(), resolvePreimage = createPreimageResolver(), -}: { - resolveAccountId?: AccountIdResolver - resolvePreimage?: PreimageResolver -} = {}) => { +}: TransactionStreamEventMapperConfig = {}) => { const mapTransactionStreamEvent = async ( ledgerTransaction: LedgerTransaction, ): Promise => { diff --git a/core/api/src/services/transactions-stream/index.ts b/core/api/src/app/transactions-stream/index.ts similarity index 80% rename from core/api/src/services/transactions-stream/index.ts rename to core/api/src/app/transactions-stream/index.ts index b2efff4d44..34019435f2 100644 --- a/core/api/src/services/transactions-stream/index.ts +++ b/core/api/src/app/transactions-stream/index.ts @@ -12,24 +12,6 @@ const toError = (err: unknown, fallbackMessage: string): Error => { return new Error(fallbackMessage) } -type SubscribeToTransactionsArgs = { - afterTransactionId?: string - onTransaction: (event: TransactionStreamEvent) => void | Promise - onError: (err: Error) => void -} - -export type TransactionsStreamSubscription = { - close: () => void -} - -type TransactionsStreamServiceConfig = { - ledgerService?: Pick - mapTransactionStreamEvent?: ( - ledgerTransaction: LedgerTransaction, - ) => Promise - logger?: Logger -} - const parseAfterTransactionId = ( afterTransactionId?: string, ): LedgerTransactionId | Error | undefined => { @@ -41,12 +23,12 @@ const parseAfterTransactionId = ( return checkedLedgerTransactionId } -export const TransactionsStreamService = ({ +export const TransactionsStream = ({ ledgerService = LedgerService(), mapTransactionStreamEvent = createTransactionStreamEventMapper() .mapTransactionStreamEvent, logger: serviceLogger = logger, -}: TransactionsStreamServiceConfig = {}) => { +}: TransactionsStreamConfig = {}) => { const subscribeToTransactions = ({ afterTransactionId, onTransaction, diff --git a/core/api/src/app/transactions-stream/index.types.d.ts b/core/api/src/app/transactions-stream/index.types.d.ts new file mode 100644 index 0000000000..27914e283a --- /dev/null +++ b/core/api/src/app/transactions-stream/index.types.d.ts @@ -0,0 +1,89 @@ +type SubscribeToTransactionsArgs = { + afterTransactionId?: string + onTransaction: (event: TransactionStreamEvent) => void | Promise + onError: (err: Error) => void +} + +type TransactionsStreamSubscription = { + close: () => void +} + +type TransactionsStreamConfig = { + ledgerService?: Pick + mapTransactionStreamEvent?: ( + ledgerTransaction: LedgerTransaction, + ) => Promise + logger?: Logger +} + +type TransactionsStreamTimestampedValue = { + value: T + expiresAt: number +} + +type TransactionsStreamExpiringCacheOptions = { + ttlMs: number + maxSize: number +} + +type TransactionsStreamPreimageLoaderArgs = { + transactionId: LedgerTransactionId + paymentHash?: PaymentHash +} + +type TransactionsStreamAccountIdLoader = ( + walletId: WalletId, +) => Promise + +type TransactionsStreamPreimageLoader = ( + args: TransactionsStreamPreimageLoaderArgs, +) => Promise + +type TransactionsStreamAccountIdResolver = ( + walletId: WalletId, +) => Promise + +type TransactionsStreamPreimageResolver = ( + args: TransactionsStreamPreimageLoaderArgs, +) => Promise + +type TransactionsStreamFindWalletById = ( + walletId: WalletId, +) => Promise<{ accountId: AccountId } | Error | undefined> + +type TransactionsStreamFindTransactionMetadataById = ( + transactionId: LedgerTransactionId, +) => Promise | null | undefined> + +type TransactionsStreamFindWalletInvoiceById = ( + paymentHash: string, +) => Promise | null | undefined> + +type TransactionsStreamAccountIdLoaderConfig = { + findWalletById?: TransactionsStreamFindWalletById +} + +type TransactionsStreamPreimageLoaderConfig = { + findTransactionMetadataById?: TransactionsStreamFindTransactionMetadataById + findWalletInvoiceById?: TransactionsStreamFindWalletInvoiceById +} + +type TransactionsStreamAccountIdResolverConfig = { + walletToAccountCache?: Map + loadAccountId?: TransactionsStreamAccountIdLoader +} + +type TransactionsStreamPreimageCache = { + get: (key: string) => string | undefined + set: (key: string, value: string) => void +} + +type TransactionsStreamPreimageResolverConfig = { + preimageCache?: TransactionsStreamPreimageCache + loadPreimage?: TransactionsStreamPreimageLoader +} + +type TransactionStreamEventMapperConfig = { + resolveAccountId?: TransactionsStreamAccountIdResolver + resolvePreimage?: TransactionsStreamPreimageResolver +} diff --git a/core/api/src/servers/transactions-grpc-stream-server.ts b/core/api/src/servers/transactions-grpc-stream-server.ts index ca17720aa3..ec34c42037 100644 --- a/core/api/src/servers/transactions-grpc-stream-server.ts +++ b/core/api/src/servers/transactions-grpc-stream-server.ts @@ -11,8 +11,8 @@ import { import { baseLogger } from "@/services/logger" import { setupMongoConnection } from "@/services/mongodb" -import { TransactionsGrpcServer } from "@/services/transactions-stream/grpc-server" -import { TransactionsStreamService } from "@/services/transactions-stream/proto/transactions_grpc_pb" +import { TransactionsGrpcServer } from "@/servers/transactions-grpc-stream/grpc-server" +import { TransactionsStreamService as TransactionsGrpcServiceDefinition } from "@/servers/transactions-grpc-stream/proto/transactions_grpc_pb" const logger = baseLogger.child({ module: "transactions-grpc-stream-server" }) @@ -43,7 +43,7 @@ const startHealthServer = async () => { const startGrpcServer = async () => { const server = new Server() - server.addService(TransactionsStreamService, TransactionsGrpcServer()) + server.addService(TransactionsGrpcServiceDefinition, TransactionsGrpcServer()) const address = `0.0.0.0:${TRANSACTIONS_GRPC_STREAM_PORT}` await new Promise((resolve, reject) => { diff --git a/core/api/src/services/transactions-stream/convert.ts b/core/api/src/servers/transactions-grpc-stream/convert.ts similarity index 100% rename from core/api/src/services/transactions-stream/convert.ts rename to core/api/src/servers/transactions-grpc-stream/convert.ts diff --git a/core/api/src/services/transactions-stream/grpc-server.ts b/core/api/src/servers/transactions-grpc-stream/grpc-server.ts similarity index 82% rename from core/api/src/services/transactions-stream/grpc-server.ts rename to core/api/src/servers/transactions-grpc-stream/grpc-server.ts index 411a07fecb..485ae09077 100644 --- a/core/api/src/services/transactions-stream/grpc-server.ts +++ b/core/api/src/servers/transactions-grpc-stream/grpc-server.ts @@ -4,10 +4,7 @@ import { transactionStreamEventToGrpcTransactionEvent } from "./convert" import { ITransactionsStreamServer } from "./proto/transactions_grpc_pb" import { SubscribeTransactionsRequest, TransactionEvent } from "./proto/transactions_pb" -import { - TransactionsStreamService, - TransactionsStreamSubscription, -} from "@/services/transactions-stream" +import { TransactionsStream } from "@/app/transactions-stream" import { baseLogger } from "@/services/logger" const logger = baseLogger.child({ module: "transactions-grpc-stream" }) @@ -19,24 +16,15 @@ const toServiceError = ({ code, message, details, -}: { - code: status - message: string - details: string -}): ServiceError => +}: TransactionsGrpcServiceErrorArgs): ServiceError => Object.assign(new Error(message), { code, details, metadata: new Metadata(), }) -type TransactionsGrpcServerConfig = { - transactionsStreamService?: ReturnType - logger?: Logger -} - export const TransactionsGrpcServer = ({ - transactionsStreamService = TransactionsStreamService(), + transactionsStream = TransactionsStream(), logger: serviceLogger = logger, }: TransactionsGrpcServerConfig = {}): ITransactionsStreamServer => { const subscribeTransactions: handleServerStreamingCall< @@ -44,7 +32,7 @@ export const TransactionsGrpcServer = ({ TransactionEvent > = (call) => { let isClosed = false - const subscriptionRef: { current?: TransactionsStreamSubscription } = {} + const subscriptionRef: TransactionsStreamSubscriptionRef = {} const cleanup = () => { if (isClosed) return @@ -66,7 +54,7 @@ export const TransactionsGrpcServer = ({ call.once("error", finish) }) - const result = transactionsStreamService.subscribeToTransactions({ + const result = transactionsStream.subscribeToTransactions({ afterTransactionId: requestAfterTransactionId(call.request), onTransaction: async (event) => { if (isClosed) return diff --git a/core/api/src/servers/transactions-grpc-stream/index.types.d.ts b/core/api/src/servers/transactions-grpc-stream/index.types.d.ts new file mode 100644 index 0000000000..af10b4aabe --- /dev/null +++ b/core/api/src/servers/transactions-grpc-stream/index.types.d.ts @@ -0,0 +1,16 @@ +type TransactionsGrpcServiceErrorArgs = { + code: import("@grpc/grpc-js").status + message: string + details: string +} + +type TransactionsStreamSubscriptionRef = { + current?: TransactionsStreamSubscription +} + +type TransactionsGrpcServerConfig = { + transactionsStream?: ReturnType< + typeof import("@/app/transactions-stream").TransactionsStream + > + logger?: Logger +} diff --git a/core/api/src/services/transactions-stream/proto/buf.gen.yaml b/core/api/src/servers/transactions-grpc-stream/proto/buf.gen.yaml similarity index 100% rename from core/api/src/services/transactions-stream/proto/buf.gen.yaml rename to core/api/src/servers/transactions-grpc-stream/proto/buf.gen.yaml diff --git a/core/api/src/services/transactions-stream/proto/transactions.proto b/core/api/src/servers/transactions-grpc-stream/proto/transactions.proto similarity index 100% rename from core/api/src/services/transactions-stream/proto/transactions.proto rename to core/api/src/servers/transactions-grpc-stream/proto/transactions.proto diff --git a/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts b/core/api/src/servers/transactions-grpc-stream/proto/transactions_grpc_pb.d.ts similarity index 100% rename from core/api/src/services/transactions-stream/proto/transactions_grpc_pb.d.ts rename to core/api/src/servers/transactions-grpc-stream/proto/transactions_grpc_pb.d.ts diff --git a/core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js b/core/api/src/servers/transactions-grpc-stream/proto/transactions_grpc_pb.js similarity index 100% rename from core/api/src/services/transactions-stream/proto/transactions_grpc_pb.js rename to core/api/src/servers/transactions-grpc-stream/proto/transactions_grpc_pb.js diff --git a/core/api/src/services/transactions-stream/proto/transactions_pb.d.ts b/core/api/src/servers/transactions-grpc-stream/proto/transactions_pb.d.ts similarity index 100% rename from core/api/src/services/transactions-stream/proto/transactions_pb.d.ts rename to core/api/src/servers/transactions-grpc-stream/proto/transactions_pb.d.ts diff --git a/core/api/src/services/transactions-stream/proto/transactions_pb.js b/core/api/src/servers/transactions-grpc-stream/proto/transactions_pb.js similarity index 100% rename from core/api/src/services/transactions-stream/proto/transactions_pb.js rename to core/api/src/servers/transactions-grpc-stream/proto/transactions_pb.js diff --git a/core/api/test/unit/services/transactions-stream/helpers.spec.ts b/core/api/test/unit/app/transactions-stream/helpers.spec.ts similarity index 99% rename from core/api/test/unit/services/transactions-stream/helpers.spec.ts rename to core/api/test/unit/app/transactions-stream/helpers.spec.ts index 597c9394b5..4be5095cda 100644 --- a/core/api/test/unit/services/transactions-stream/helpers.spec.ts +++ b/core/api/test/unit/app/transactions-stream/helpers.spec.ts @@ -20,7 +20,7 @@ import { createPreimageLoader, createPreimageResolver, createTransactionStreamEventMapper, -} from "@/services/transactions-stream/helpers" +} from "@/app/transactions-stream/helpers" import { TransactionMetadata } from "@/services/ledger/schema" import { diff --git a/core/api/test/unit/services/transactions-stream/index.spec.ts b/core/api/test/unit/app/transactions-stream/index.spec.ts similarity index 93% rename from core/api/test/unit/services/transactions-stream/index.spec.ts rename to core/api/test/unit/app/transactions-stream/index.spec.ts index 4be0968ea4..346843adb4 100644 --- a/core/api/test/unit/services/transactions-stream/index.spec.ts +++ b/core/api/test/unit/app/transactions-stream/index.spec.ts @@ -20,7 +20,7 @@ jest.mock("@/services/mongoose/wallets", () => ({ WalletsRepository: jest.fn(), })) -import { TransactionsStreamService } from "@/services/transactions-stream" +import { TransactionsStream } from "@/app/transactions-stream" import { TransactionsStreamSettlementVia, @@ -87,7 +87,7 @@ const subscribe = ({ service, afterTransactionId, }: { - service: ReturnType + service: ReturnType afterTransactionId?: string }) => { const onTransaction = jest.fn() @@ -101,12 +101,12 @@ const subscribe = ({ return { onTransaction, onError, subscription } } -describe("TransactionsStreamService", () => { +describe("TransactionsStream", () => { it("returns an error for malformed cursors", () => { const ledgerService = { streamSettledTransactions: jest.fn(), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn(), }) @@ -124,7 +124,7 @@ describe("TransactionsStreamService", () => { const ledgerService = { streamSettledTransactions: jest.fn(), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn(), }) @@ -151,7 +151,7 @@ describe("TransactionsStreamService", () => { .mockImplementation(async (txn: LedgerTransaction) => createEvent(txn.id), ) - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent, }) @@ -186,7 +186,7 @@ describe("TransactionsStreamService", () => { .fn() .mockReturnValue(ledgerTransactionGenerator([])), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn(), }) @@ -208,7 +208,7 @@ describe("TransactionsStreamService", () => { .fn() .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn().mockResolvedValue(undefined), }) @@ -227,7 +227,7 @@ describe("TransactionsStreamService", () => { .fn() .mockReturnValue(ledgerTransactionGenerator([ledgerError])), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn(), logger: { error: jest.fn() } as unknown as Logger, @@ -247,7 +247,7 @@ describe("TransactionsStreamService", () => { .fn() .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn().mockRejectedValue(mapperError), logger: { error: jest.fn() } as unknown as Logger, @@ -274,7 +274,7 @@ describe("TransactionsStreamService", () => { })() }), } - const service = TransactionsStreamService({ + const service = TransactionsStream({ ledgerService, mapTransactionStreamEvent: jest.fn(), }) diff --git a/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts b/core/api/test/unit/servers/transactions-grpc-stream/grpc-server.spec.ts similarity index 85% rename from core/api/test/unit/services/transactions-stream/grpc-server.spec.ts rename to core/api/test/unit/servers/transactions-grpc-stream/grpc-server.spec.ts index 5e066efaba..edf7a84719 100644 --- a/core/api/test/unit/services/transactions-stream/grpc-server.spec.ts +++ b/core/api/test/unit/servers/transactions-grpc-stream/grpc-server.spec.ts @@ -24,9 +24,9 @@ jest.mock("@/services/mongoose/wallets", () => ({ import { status } from "@grpc/grpc-js" -import { TransactionsStreamService } from "@/services/transactions-stream" -import { TransactionsGrpcServer } from "@/services/transactions-stream/grpc-server" -import { SubscribeTransactionsRequest } from "@/services/transactions-stream/proto/transactions_pb" +import { TransactionsStream } from "@/app/transactions-stream" +import { TransactionsGrpcServer } from "@/servers/transactions-grpc-stream/grpc-server" +import { SubscribeTransactionsRequest } from "@/servers/transactions-grpc-stream/proto/transactions_pb" import { LedgerTransactionType } from "@/domain/ledger" import { @@ -98,11 +98,11 @@ async function* ledgerTransactionGenerator(values: LedgerTransaction { it("returns INVALID_ARGUMENT for malformed cursors", () => { - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockReturnValue(new Error("invalid cursor")), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("not-an-object-id") @@ -110,7 +110,7 @@ describe("TransactionsGrpcServer", () => { expect(call.destroy).toHaveBeenCalledTimes(1) expect(call.destroy.mock.calls[0][0].code).toBe(status.INVALID_ARGUMENT) - expect(transactionsStreamService.subscribeToTransactions).toHaveBeenCalledWith({ + expect(transactionsStream.subscribeToTransactions).toHaveBeenCalledWith({ afterTransactionId: "not-an-object-id", onTransaction: expect.any(Function), onError: expect.any(Function), @@ -118,11 +118,11 @@ describe("TransactionsGrpcServer", () => { }) it("returns INVALID_ARGUMENT for explicitly empty cursors", () => { - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockReturnValue(new Error("invalid cursor")), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("") @@ -130,7 +130,7 @@ describe("TransactionsGrpcServer", () => { expect(call.destroy).toHaveBeenCalledTimes(1) expect(call.destroy.mock.calls[0][0].code).toBe(status.INVALID_ARGUMENT) - expect(transactionsStreamService.subscribeToTransactions).toHaveBeenCalledWith({ + expect(transactionsStream.subscribeToTransactions).toHaveBeenCalledWith({ afterTransactionId: "", onTransaction: expect.any(Function), onError: expect.any(Function), @@ -139,14 +139,14 @@ describe("TransactionsGrpcServer", () => { it("maps domain events to grpc messages", async () => { const close = jest.fn() - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockImplementation(({ onTransaction }) => { onTransaction(createEvent()) return { close } }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("661111111111111111111110") @@ -176,11 +176,11 @@ describe("TransactionsGrpcServer", () => { .mockReturnValue(ledgerTransactionGenerator([ledgerTransaction])), } const mapTransactionStreamEvent = jest.fn().mockResolvedValue(createEvent()) - const transactionsStreamService = TransactionsStreamService({ + const transactionsStream = TransactionsStream({ ledgerService, mapTransactionStreamEvent, }) - const grpcServer = TransactionsGrpcServer({ transactionsStreamService }) + const grpcServer = TransactionsGrpcServer({ transactionsStream }) const call = createCall("000000000000000000000000") grpcServer.subscribeTransactions(call as never) @@ -197,11 +197,11 @@ describe("TransactionsGrpcServer", () => { it("does not close the subscription on grpc close", () => { const close = jest.fn() - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockReturnValue({ close }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("661111111111111111111110") @@ -213,11 +213,11 @@ describe("TransactionsGrpcServer", () => { it("closes the subscription on client cancellation", () => { const close = jest.fn() - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockReturnValue({ close }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("661111111111111111111110") @@ -229,11 +229,11 @@ describe("TransactionsGrpcServer", () => { it("closes the subscription on grpc call errors", () => { const close = jest.fn() - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockReturnValue({ close }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("661111111111111111111110") @@ -243,18 +243,18 @@ describe("TransactionsGrpcServer", () => { expect(close).toHaveBeenCalledTimes(1) }) - it("destroys the grpc call on service stream errors", () => { + it("destroys the grpc call on app stream errors", () => { const close = jest.fn() const streamError = new Error("change stream failed") let onError: ((err: Error) => void) | undefined - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockImplementation(({ onError: handler }) => { onError = handler return { close } }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, logger: { error: jest.fn() } as unknown as Logger, }) const call = createCall("661111111111111111111110") @@ -268,14 +268,14 @@ describe("TransactionsGrpcServer", () => { it("waits for grpc drain when writes apply backpressure", async () => { let onTransactionResult: Promise | undefined - const transactionsStreamService = { + const transactionsStream = { subscribeToTransactions: jest.fn().mockImplementation(({ onTransaction }) => { onTransactionResult = onTransaction(createEvent()) return { close: jest.fn() } }), } const grpcServer = TransactionsGrpcServer({ - transactionsStreamService: transactionsStreamService as never, + transactionsStream: transactionsStream as never, }) const call = createCall("661111111111111111111110") call.write.mockReturnValueOnce(false) From 143a0f8a7405ab3a979f3beb54e31b0f9f8dc30d Mon Sep 17 00:00:00 2001 From: basarrcan Date: Wed, 6 May 2026 04:55:19 +0300 Subject: [PATCH 08/11] refactor: broadcast all settled liabilities transactions --- .../ledger/stream-settled-transactions.ts | 52 ------------------- .../stream-settled-transactions.types.d.ts | 30 +++++++++++ .../stream-settled-transactions.spec.ts | 37 +++---------- 3 files changed, 38 insertions(+), 81 deletions(-) create mode 100644 core/api/src/services/ledger/stream-settled-transactions.types.d.ts diff --git a/core/api/src/services/ledger/stream-settled-transactions.ts b/core/api/src/services/ledger/stream-settled-transactions.ts index f4478ce674..43edcc572c 100644 --- a/core/api/src/services/ledger/stream-settled-transactions.ts +++ b/core/api/src/services/ledger/stream-settled-transactions.ts @@ -1,57 +1,11 @@ import { translateToLedgerTx } from "./translate" -import { LedgerTransactionType } from "@/domain/ledger" import { UnknownLedgerError } from "@/domain/ledger/errors" -import { WalletCurrency } from "@/domain/shared" import { toObjectId } from "@/services/mongoose/utils" const SETTLED_TRANSACTION_STREAM_BATCH_SIZE = 100 const DEFAULT_REPLAY_DEDUPE_CACHE_SIZE = 10_000 const LIABILITIES_ACCOUNT_PATTERN = /^Liabilities:/ -const EXCLUDED_SETTLED_TRANSACTION_TYPES: LedgerTransactionType[] = [ - LedgerTransactionType.Fee, - LedgerTransactionType.ToColdStorage, - LedgerTransactionType.ToHotWallet, - LedgerTransactionType.Escrow, - LedgerTransactionType.RoutingRevenue, - LedgerTransactionType.Reconciliation, -] -const SETTLED_TRANSACTION_CHANGE_OPERATIONS = ["insert", "replace", "update"] -const SETTLED_TRANSACTION_CHANGE_OPERATION_MATCH = [ - { operationType: { $in: ["insert", "replace"] } }, - { "updateDescription.updatedFields.pending": false }, -] - -type TransactionCursor = AsyncIterable & { - close: () => Promise -} - -type TransactionFindQuery = { - sort: (sort: Record) => { - cursor: (options: { batchSize: number }) => TransactionCursor - } -} - -type TransactionChangeStream = { - next: () => Promise - close: () => Promise -} - -type SettledTransactionModel = { - find: (filter: Record) => TransactionFindQuery - watch: ( - pipeline: Record[], - options: { fullDocument: "updateLookup" }, - ) => TransactionChangeStream -} - -type StreamSettledTransactionsConfig = { - transactionModel: SettledTransactionModel - translateLedgerTransaction?: ( - tx: ILedgerTransaction, - ) => LedgerTransaction - maxReplayDedupeCacheSize?: number -} export const settledTransactionFilter = ( afterTransactionId?: LedgerTransactionId, @@ -59,8 +13,6 @@ export const settledTransactionFilter = ( const filter: Record = { accounts: LIABILITIES_ACCOUNT_PATTERN, pending: false, - voided: { $ne: true }, - type: { $nin: EXCLUDED_SETTLED_TRANSACTION_TYPES }, } if (afterTransactionId) { @@ -74,12 +26,8 @@ export const settledTransactionChangeStreamPipeline = ( afterTransactionId?: LedgerTransactionId, ): Record[] => { const match: Record = { - "operationType": { $in: SETTLED_TRANSACTION_CHANGE_OPERATIONS }, - "$or": SETTLED_TRANSACTION_CHANGE_OPERATION_MATCH, "fullDocument.accounts": LIABILITIES_ACCOUNT_PATTERN, "fullDocument.pending": false, - "fullDocument.voided": { $ne: true }, - "fullDocument.type": { $nin: EXCLUDED_SETTLED_TRANSACTION_TYPES }, } if (afterTransactionId) { diff --git a/core/api/src/services/ledger/stream-settled-transactions.types.d.ts b/core/api/src/services/ledger/stream-settled-transactions.types.d.ts new file mode 100644 index 0000000000..2f2e987cc6 --- /dev/null +++ b/core/api/src/services/ledger/stream-settled-transactions.types.d.ts @@ -0,0 +1,30 @@ +type SettledTransactionCursor = AsyncIterable & { + close: () => Promise +} + +type SettledTransactionFindQuery = { + sort: (sort: Record) => { + cursor: (options: { batchSize: number }) => SettledTransactionCursor + } +} + +type SettledTransactionChangeStream = { + next: () => Promise + close: () => Promise +} + +type SettledTransactionModel = { + find: (filter: Record) => SettledTransactionFindQuery + watch: ( + pipeline: Record[], + options: { fullDocument: "updateLookup" }, + ) => SettledTransactionChangeStream +} + +type StreamSettledTransactionsConfig = { + transactionModel: SettledTransactionModel + translateLedgerTransaction?: ( + tx: ILedgerTransaction, + ) => LedgerTransaction + maxReplayDedupeCacheSize?: number +} diff --git a/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts b/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts index ad3f5a62bc..b415c2b530 100644 --- a/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts +++ b/core/api/test/unit/services/ledger/stream-settled-transactions.spec.ts @@ -91,7 +91,7 @@ const createTransactionModel = ({ } describe("settled transaction stream query helpers", () => { - it("filters settled customer transactions after a cursor", () => { + it("filters settled liabilities transactions after a cursor", () => { const filter = settledTransactionFilter( "661111111111111111111111" as LedgerTransactionId, ) @@ -99,24 +99,15 @@ describe("settled transaction stream query helpers", () => { expect(filter).toMatchObject({ accounts: /^Liabilities:/, pending: false, - voided: { $ne: true }, - type: { - $nin: [ - LedgerTransactionType.Fee, - LedgerTransactionType.ToColdStorage, - LedgerTransactionType.ToHotWallet, - LedgerTransactionType.Escrow, - LedgerTransactionType.RoutingRevenue, - LedgerTransactionType.Reconciliation, - ], - }, }) + expect(filter).not.toHaveProperty("voided") + expect(filter).not.toHaveProperty("type") expect((filter._id as { $gt: mongoose.Types.ObjectId }).$gt.toString()).toBe( "661111111111111111111111", ) }) - it("builds the same constraints for live change streams", () => { + it("builds liabilities constraints for live change streams", () => { const pipeline = settledTransactionChangeStreamPipeline( "661111111111111111111111" as LedgerTransactionId, ) @@ -124,30 +115,18 @@ describe("settled transaction stream query helpers", () => { expect(pipeline).toEqual([ { $match: { - "operationType": { $in: ["insert", "replace", "update"] }, - "$or": [ - { operationType: { $in: ["insert", "replace"] } }, - { "updateDescription.updatedFields.pending": false }, - ], "fullDocument.accounts": /^Liabilities:/, "fullDocument.pending": false, - "fullDocument.voided": { $ne: true }, - "fullDocument.type": { - $nin: [ - LedgerTransactionType.Fee, - LedgerTransactionType.ToColdStorage, - LedgerTransactionType.ToHotWallet, - LedgerTransactionType.Escrow, - LedgerTransactionType.RoutingRevenue, - LedgerTransactionType.Reconciliation, - ], - }, "fullDocument._id": { $gt: new mongoose.Types.ObjectId("661111111111111111111111"), }, }, }, ]) + expect(pipeline[0].$match).not.toHaveProperty("fullDocument.voided") + expect(pipeline[0].$match).not.toHaveProperty("fullDocument.type") + expect(pipeline[0].$match).not.toHaveProperty("operationType") + expect(pipeline[0].$match).not.toHaveProperty("$or") }) }) From b45e921016da4c658076e86b8ecc30ce46f73d6f Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 7 May 2026 05:55:24 +0300 Subject: [PATCH 09/11] chore(quickstart): add transaction grpc stream service --- ci/core/pipeline.yml | 5 +++-- ci/core/tasks/open-core-bundle-charts-pr.sh | 2 +- ci/core/tasks/prep-quickstart.sh | 2 ++ ci/core/template.lib.yml | 2 +- quickstart/docker-compose.tmpl.yml | 19 +++++++++++++++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ci/core/pipeline.yml b/ci/core/pipeline.yml index 7acdbe6072..ec7e036032 100644 --- a/ci/core/pipeline.yml +++ b/ci/core/pipeline.yml @@ -250,11 +250,11 @@ jobs: - get: #@ component_src_resource_name("api") trigger: true passed: - #@ for component in ["api", "api-migrate", "api-ws-server", "api-trigger"]: + #@ for component in ["api", "api-migrate", "api-ws-server", "api-trigger", "api-transactions-grpc-stream"]: - #@ build_edge_image_name(component) #@ end - get: pipeline-tasks - #@ for component in ["api", "api-migrate", "api-ws-server", "api-trigger", "notifications"]: + #@ for component in ["api", "api-migrate", "api-ws-server", "api-trigger", "api-transactions-grpc-stream", "notifications"]: - get: #@ edge_image_resource_name(component) passed: - #@ build_edge_image_name(component) @@ -272,6 +272,7 @@ jobs: - name: api-edge-image - name: api-migrate-edge-image - name: api-trigger-edge-image + - name: api-transactions-grpc-stream-edge-image - name: notifications-edge-image - name: pipeline-tasks outputs: diff --git a/ci/core/tasks/open-core-bundle-charts-pr.sh b/ci/core/tasks/open-core-bundle-charts-pr.sh index 299d610d5f..c137de0468 100755 --- a/ci/core/tasks/open-core-bundle-charts-pr.sh +++ b/ci/core/tasks/open-core-bundle-charts-pr.sh @@ -33,7 +33,7 @@ git remote set-url origin ${github_url} git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git checkout ${ref} -app_src_files=($(buck2 uquery 'inputs(deps("//core/api:")) + inputs(deps("//core/api-ws-server:")) + inputs(deps("//core/api-trigger:")) + inputs(deps("//core/api-exporter:")) + inputs(deps("//core/api-cron:"))' 2>/dev/null)) +app_src_files=($(buck2 uquery 'inputs(deps("//core/api:")) + inputs(deps("//core/api-ws-server:")) + inputs(deps("//core/api-trigger:")) + inputs(deps("//core/api-transactions-grpc-stream:")) + inputs(deps("//core/api-exporter:")) + inputs(deps("//core/api-cron:"))' 2>/dev/null)) declare -A relevant_commits relevant_commits=() diff --git a/ci/core/tasks/prep-quickstart.sh b/ci/core/tasks/prep-quickstart.sh index aa62846f6d..17e455f95a 100755 --- a/ci/core/tasks/prep-quickstart.sh +++ b/ci/core/tasks/prep-quickstart.sh @@ -2,6 +2,7 @@ export api_digest=$(cat ./api-edge-image/digest | sed 's/:/@/') export trigger_digest=$(cat ./api-trigger-edge-image/digest | sed 's/:/@/') +export transactions_grpc_stream_digest=$(cat ./api-transactions-grpc-stream-edge-image/digest | sed 's/:/@/') export notifications_digest=$(cat ./notifications-edge-image/digest | sed 's/:/@/') export migrate_digest=$(cat ./api-migrate-edge-image/digest | sed 's/:/@/') @@ -11,6 +12,7 @@ pushd repo/quickstart || exit 1 ./bin/bump-galoy-image-digest.sh "api" "$api_digest" ./bin/bump-galoy-image-digest.sh "trigger" "$trigger_digest" +./bin/bump-galoy-image-digest.sh "transactions_grpc_stream" "$transactions_grpc_stream_digest" ./bin/bump-galoy-image-digest.sh "notifications" "$notifications_digest" ./bin/bump-mongodb-migrate-image-digest.sh "$migrate_digest" ./bin/re-render.sh diff --git a/ci/core/template.lib.yml b/ci/core/template.lib.yml index 0afe078eb8..8f45105990 100644 --- a/ci/core/template.lib.yml +++ b/ci/core/template.lib.yml @@ -1,6 +1,6 @@ #@ load("@ytt:data", "data") -#@ core_bundle_components = ["api", "api-cron", "api-trigger", "api-ws-server", "api-exporter"] +#@ core_bundle_components = ["api", "api-cron", "api-trigger", "api-transactions-grpc-stream", "api-ws-server", "api-exporter"] #@ def galoy_dev_image(): #@ return data.values.docker_registry + "/galoy-dev" diff --git a/quickstart/docker-compose.tmpl.yml b/quickstart/docker-compose.tmpl.yml index d3746f8321..4b9b163426 100644 --- a/quickstart/docker-compose.tmpl.yml +++ b/quickstart/docker-compose.tmpl.yml @@ -2,6 +2,7 @@ #@ galoy_api_image_digest = "sha256@99cb85d077212f1ff3ff69349be05cd065fd5751d322ff4f9bbad9f7b8bc4042" #@ galoy_trigger_image_digest = "sha256@724ae64a8bcb0fcc6f1fede31a8345937e39ebd5e17ec274e2d857e53cd22fa0" +#@ galoy_transactions_grpc_stream_image_digest = "edge" #@ mongodb_migrate_image_digest = "sha256@1ef27178b91fd42d330c90fd1e88743f2f9893628a1b6d514c9b99689215b406" #@ galoy_notifications_image_digest = "sha256@5eae5078c192c8f4aa3a9bed8524fa1ea8fac340a2d56374401acb9db0bdf6b0" #@ galoy_api_keys_image_digest = "sha256@4b0b7bee8441dfaf623a1693dd60e24669db0b9e37baa30ecd8925cbe6590a1e" @@ -56,6 +57,7 @@ services: - kratos - galoy - trigger + - transactions-grpc-stream - redis - mongodb - mongodb-migrate @@ -127,6 +129,23 @@ services: depends_on: - galoy + transactions-grpc-stream: +#@ if galoy_transactions_grpc_stream_image_digest == "local": + build: + context: .. + dockerfile: core/api-transactions-grpc-stream/Dockerfile +#@ elif galoy_transactions_grpc_stream_image_digest.startswith("sha256@"): + image: #@ "us.gcr.io/galoy-org/galoy-api-transactions-grpc-stream@" + galoy_transactions_grpc_stream_image_digest.replace("@",":") +#@ else: + image: #@ "us.gcr.io/galoy-org/galoy-api-transactions-grpc-stream:" + galoy_transactions_grpc_stream_image_digest +#@ end + environment: #@ core_env + ports: + - 50053:50053 + - 8889:8889 + depends_on: + - mongodb + notifications: #@ if galoy_notifications_image_digest == "local": build: From 7c1245a673c76a5b7ec265c2070b160f8113fa44 Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 7 May 2026 06:07:21 +0300 Subject: [PATCH 10/11] fix: package transaction stream server protos --- core/api/BUCK | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/api/BUCK b/core/api/BUCK index c79068a38e..7233f96618 100644 --- a/core/api/BUCK +++ b/core/api/BUCK @@ -63,7 +63,8 @@ filegroup( name = "protos", srcs = glob([ "src/services/**/protos/**", - "src/services/**/proto/**" + "src/services/**/proto/**", + "src/servers/**/proto/**", ]) ) From 67fa0cb4496b9e0b6adcb68ac7956a1142b3b816 Mon Sep 17 00:00:00 2001 From: basarrcan Date: Thu, 7 May 2026 06:51:09 +0300 Subject: [PATCH 11/11] chore(quickstart): sync template digests with main --- quickstart/docker-compose.tmpl.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quickstart/docker-compose.tmpl.yml b/quickstart/docker-compose.tmpl.yml index 4b9b163426..dc593c4094 100644 --- a/quickstart/docker-compose.tmpl.yml +++ b/quickstart/docker-compose.tmpl.yml @@ -1,11 +1,11 @@ #@ load("@ytt:data", "data") -#@ galoy_api_image_digest = "sha256@99cb85d077212f1ff3ff69349be05cd065fd5751d322ff4f9bbad9f7b8bc4042" -#@ galoy_trigger_image_digest = "sha256@724ae64a8bcb0fcc6f1fede31a8345937e39ebd5e17ec274e2d857e53cd22fa0" -#@ galoy_transactions_grpc_stream_image_digest = "edge" -#@ mongodb_migrate_image_digest = "sha256@1ef27178b91fd42d330c90fd1e88743f2f9893628a1b6d514c9b99689215b406" -#@ galoy_notifications_image_digest = "sha256@5eae5078c192c8f4aa3a9bed8524fa1ea8fac340a2d56374401acb9db0bdf6b0" +#@ galoy_api_image_digest = "sha256@f9ced5eb7b5ceca763208c21479af4a071debe507b96f2ada52471e99598ffc4" +#@ galoy_trigger_image_digest = "sha256@229535bfed9d1f7d9fc754e1d4390d92d445abc2740a7fb1b9507b7ea2d175f5" +#@ mongodb_migrate_image_digest = "sha256@be2ae46118bef496902b7958d98fabb18447f73b3683e46324ad6679a8ef5824" +#@ galoy_notifications_image_digest = "sha256@afd81cd792dee3d51d0fef4ab73c4326afed3fae1a834329204f142b7405c1a7" #@ galoy_api_keys_image_digest = "sha256@4b0b7bee8441dfaf623a1693dd60e24669db0b9e37baa30ecd8925cbe6590a1e" +#@ galoy_transactions_grpc_stream_image_digest = "edge" #@ core_env = [ #@ "HELMREVISION=dev",