From 541941e3cd39a9fddab345211937f1a5799223ec Mon Sep 17 00:00:00 2001 From: mainqueg Date: Thu, 18 Dec 2025 17:33:52 -0300 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=9A=A7=20server:=20kyc=20account=20mi?= =?UTF-8?q?gration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/good-apples-arrive.md | 2 + server/script/kyc-migration.ts | 304 ++++++++++++++++++++++++++++++ server/test/api/kyc.test.ts | 2 + server/test/utils/persona.test.ts | 58 +++--- server/utils/persona.ts | 176 ++++++++++++++++- server/utils/validatorHook.ts | 2 +- 6 files changed, 521 insertions(+), 23 deletions(-) create mode 100644 .changeset/good-apples-arrive.md create mode 100644 server/script/kyc-migration.ts diff --git a/.changeset/good-apples-arrive.md b/.changeset/good-apples-arrive.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/good-apples-arrive.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/server/script/kyc-migration.ts b/server/script/kyc-migration.ts new file mode 100644 index 000000000..a50d008d0 --- /dev/null +++ b/server/script/kyc-migration.ts @@ -0,0 +1,304 @@ +import createDebug from "debug"; +import { inspect } from "node:util"; +import { safeParse, ValiError, type InferOutput } from "valibot"; + +import * as persona from "../utils/persona"; +import { buildIssueMessages } from "../utils/validatorHook"; + +const BATCH_SIZE = 10; + +const debug = createDebug("migration:debug"); +const log = createDebug("migration:log"); +const warn = createDebug("migration:warn"); +const unexpected = createDebug("migration:unexpected"); + +let reference: string | undefined; +let all = false; +let onlyLogs = false; +let initialNext: string | undefined; +let onlyPandaTemplates = false; + +const options = process.argv.slice(2); +for (const option of options) { + switch (true) { + case option.startsWith("--reference-id="): + reference = option.split("=")[1]; + break; + case option.startsWith("--all"): + all = true; + break; + case option.startsWith("--only-logs"): + log("Running in only logs mode"); + onlyLogs = true; + break; + case option.startsWith("--next="): + initialNext = option.split("=")[1]; + break; + case option.startsWith("--only-panda-templates"): + onlyPandaTemplates = true; + break; + default: + unexpected(`❌ unknown option: ${option}`); + throw new Error(`unknown option: ${option}`); + } +} + +main().catch((error: unknown) => { + unexpected("❌ migration failed", inspect(error, { depth: null, colors: true })); +}); + +let migratedAccounts = 0; +let redactedAccounts = 0; +let redactedInquiries = 0; +let failedToRedactAccounts = 0; +let noApprovedInquiryAccounts = 0; +let unknownTemplates = 0; +let cryptomateTemplates = 0; +let pandaTemplates = 0; +let schemaErrors = 0; +let failedAccounts = 0; +let inquirySchemaErrors = 0; +let noReferenceIdAccounts = 0; +let totalAccounts = 0; + +async function main() { + if (all) { + log("🔍 Processing all accounts"); + } else if (reference) { + log(`🔍 Processing accounts with reference ID: ${reference}`); + } else { + unexpected("❌ please provide --reference-id= or --all is required"); + throw new Error("missing --reference-id= or --all"); + } + + let next = initialNext; + let batch = 0; + let retries = 0; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { + try { + log( + `\n ----- Processing batch ${batch++} (Batch size: ${BATCH_SIZE}, next: ${next ?? "undefined"}) ${retries > 0 ? `(Retry ${retries})` : ""} -----`, + ); + + const accounts = await getAccounts(BATCH_SIZE, next ?? undefined, reference).catch((error: unknown) => { + if (error instanceof ValiError) { + unexpected(`❌ Failed process batch ${batch} due to schema errors. Aborting...`); + unexpected("❌ Schema errors:", buildIssueMessages(error.issues)); + return { data: [], links: { next: null } }; + } + throw error; + }); + + totalAccounts += accounts.data.length; + log(`🔍 Found ${accounts.data.length} accounts`); + + for (const account of accounts.data) { + try { + if (!account.attributes["reference-id"]) { + noReferenceIdAccounts++; + warn(`Account ${account.id} has no reference id`); + continue; + } + await processAccount(account.id, account.attributes["reference-id"]); + } catch (error: unknown) { + unexpected( + `❌ Failed to process batch ${batch}, next: ${next ?? "undefined"}, account: ${account.id}/${account.attributes["reference-id"]} due to: ${inspect(error, { depth: null, colors: true })}`, + ); + failedAccounts++; + await new Promise((resolve) => setTimeout(resolve, 1000)); + continue; + } + } + + next = accounts.data.at(-1)?.id; + if (!next) break; + retries = 0; + } catch (error: unknown) { + unexpected(`❌ Failed to process batch ${batch} due to: ${inspect(error, { depth: null, colors: true })}`); + await new Promise((resolve) => setTimeout(resolve, 1000)); + retries++; + if (retries >= 3) { + unexpected(`❌ Failed to process batch ${batch} after 3 retries. Aborting...`); + break; + } + } + } + + log(`\n ----- Migration summary -----`); + log(`🔍 Total accounts processed: ${totalAccounts}`); + log(`🔍 Redacted inquiries: ${redactedInquiries}`); + log(`🔍 No approved inquiry accounts, redaction needed: ${noApprovedInquiryAccounts}`); + log(` ---------------------------------`); + log(`✅ Migrated approved accounts: ${migratedAccounts}`); + log(`♻️ Redacted accounts: ${redactedAccounts}`); + log(`❌ Accounts failed to redact: ${failedToRedactAccounts}`); + log(`❌ Accounts failed to process: ${failedAccounts}`); + log(` ---------------------------------`); + + log(`\n ----- Approved accounts summary -----`); + log(`🔍 Panda templates: ${pandaTemplates}`); + log(`🚨 Schema errors: ${schemaErrors}`); + log(`🚨 Inquiry schema errors: ${inquirySchemaErrors}`); + log(` ---------------------------------`); + + log(`\n ----- Inquiry Statistics summary -----`); + log(`🚨 Unknown templates: ${unknownTemplates}`); + log(`⚰️ Cryptomate templates: ${cryptomateTemplates}`); + log(`⚠️ No reference id accounts: ${noReferenceIdAccounts}`); + log(` ----- Statistics summary -----`); +} + +function getAccounts(limit: number, after?: string, referenceId?: string) { + return persona.getUnknownAccounts(limit, after, referenceId); +} + +function updateAccountFromInquiry(accountId: string, inquiry: InferOutput) { + const annualSalary = + inquiry.attributes.fields["annual-salary-ranges-us-150-000"]?.value ?? + inquiry.attributes.fields["annual-salary"]?.value; + const expectedMonthlyVolume = + inquiry.attributes.fields["monthly-purchases-range"]?.value ?? + inquiry.attributes.fields["expected-monthly-volume"]?.value; + if (!annualSalary) throw new Error("annual salary is required"); + if (!expectedMonthlyVolume) throw new Error("expected monthly volume is required"); + + const exaCardTc = + inquiry.attributes.fields["new-screen-2-2-input-checkbox"]?.value ?? + inquiry.attributes.fields["new-screen-input-checkbox-2"]?.value; + if (exaCardTc !== true) throw new Error("exa card tc is required"); + + return persona.updateAccount(accountId, { + rain_e_sign_consent: inquiry.attributes.fields["input-checkbox"].value, + exa_card_tc: exaCardTc, + privacy__policy: inquiry.attributes.fields["new-screen-input-checkbox"].value, + account_opening_disclosure: inquiry.attributes.fields["new-screen-input-checkbox-4"]?.value ?? null, + + economic_activity: inquiry.attributes.fields["input-select"].value, + annual_salary: annualSalary, + expected_monthly_volume: expectedMonthlyVolume, + accurate_info_confirmation: inquiry.attributes.fields["new-screen-input-checkbox-1"].value, + non_unauthorized_solicitation: inquiry.attributes.fields["new-screen-input-checkbox-3"].value, + non_illegal_activities_2: inquiry.attributes.fields["illegal-activites"].value, // cspell:ignore illegal-activites + }); +} + +async function processAccount(accountId: string, referenceId: string) { + const unknownInquiry = await persona + .getUnknownApprovedInquiry(referenceId, onlyPandaTemplates ? persona.PANDA_TEMPLATE : undefined) + .catch((error: unknown) => { + if (error instanceof ValiError) { + unexpected( + `❌ Failed to get unknown approved inquiry for account ${referenceId}/${accountId} due to schema errors`, + buildIssueMessages(error.issues), + ); + inquirySchemaErrors++; + throw error; + } + throw error; + }); + + if (!unknownInquiry) { + noApprovedInquiryAccounts++; + log(`Account ${referenceId}/${accountId} has no approved inquiry. Redacting account...`); + if (onlyLogs) return; + await persona + .redactAccount(accountId) + .then(() => { + log(`♻️ Account ${referenceId}/${accountId} redacted successfully`); + redactedAccounts++; + }) + .catch((error: unknown) => { + unexpected( + `❌ Account ${referenceId}/${accountId} redacting failed`, + inspect(error, { depth: null, colors: true }), + ); + failedToRedactAccounts++; + }); + return; + } + + if (unknownInquiry.attributes["redacted-at"]) { + redactedInquiries++; + log(`Inquiry ${referenceId}/${accountId} is redacted. Redacting account...`); + if (onlyLogs) return; + await persona + .redactAccount(accountId) + .then(() => { + log(`♻️ Account ${referenceId}/${accountId} redacted successfully`); + redactedAccounts++; + }) + .catch((error: unknown) => { + unexpected( + `❌ Account ${referenceId}/${accountId} redacting failed`, + inspect(error, { depth: null, colors: true }), + ); + failedToRedactAccounts++; + }); + return; + } + + const isPandaTemplate = unknownInquiry.relationships["inquiry-template"]?.data.id === persona.PANDA_TEMPLATE; + const isCryptomateTemplate = + unknownInquiry.relationships["inquiry-template"]?.data.id === persona.CRYPTOMATE_TEMPLATE; + + if (isPandaTemplate) { + pandaTemplates++; + const pandaInquiry = safeParse(persona.PandaInquiryApproved, unknownInquiry); + if (!pandaInquiry.success) { + inquirySchemaErrors++; + unexpected( + `❌ Account ${referenceId}/${accountId} failed to parse panda inquiry`, + buildIssueMessages(pandaInquiry.issues), + ); + return; + } + debug(`✅ PANDA TEMPLATE: Account ${referenceId}/${accountId} has approved inquiry`); + if (onlyLogs) return; + await updateAccountFromInquiry(accountId, pandaInquiry.output); + await persona.addDocument(pandaInquiry.output.attributes["reference-id"], { + id_class: { value: pandaInquiry.output.attributes.fields["identification-class"].value }, + id_number: { value: pandaInquiry.output.attributes.fields["identification-number"].value }, + id_issuing_country: { value: pandaInquiry.output.attributes.fields["selected-country-code"].value }, + id_document_id: { value: pandaInquiry.output.attributes.fields["current-government-id"].value.id }, + }); + + // validate basic scope + const basicAccount = await persona.getAccount(referenceId, "basic").catch((error: unknown) => { + if (error instanceof ValiError) { + schemaErrors++; + unexpected( + `❌ Account ${referenceId}/${accountId} failed to get basic scope due to schema errors`, + buildIssueMessages(error.issues), + ); + } else { + failedAccounts++; + unexpected( + `❌ Account ${referenceId}/${accountId} getting basic scope failed`, + inspect(error, { depth: null, colors: true }), + ); + } + }); + + if (!basicAccount) { + unexpected(`❌ Account ${referenceId}/${accountId} failed to get basic scope`); + return; + } + log(`🎉 PANDA TEMPLATE: Account ${referenceId}/${basicAccount.id} has been migrated and has a valid basic scope`); + migratedAccounts++; + return; + } + + if (isCryptomateTemplate) { + cryptomateTemplates++; + warn( + `⚰️ CRYPTOMATE TEMPLATE: Account ${referenceId} has approved inquiry of template ${unknownInquiry.relationships["inquiry-template"]?.data.id}`, + ); + return; + } + + unknownTemplates++; + warn( + `🚨 UNKNOWN TEMPLATE: Account ${referenceId} has an approved inquiry of template ${unknownInquiry.relationships["inquiry-template"]?.data.id}`, + ); +} diff --git a/server/test/api/kyc.test.ts b/server/test/api/kyc.test.ts index fe1a3dc23..8d6d1fe41 100644 --- a/server/test/api/kyc.test.ts +++ b/server/test/api/kyc.test.ts @@ -1057,6 +1057,7 @@ const personaTemplate = { attributes: { status: "approved" as const, "reference-id": "ref-123", + "redacted-at": null, "name-first": "John", "name-middle": null, "name-last": "Doe", @@ -1068,6 +1069,7 @@ const personaTemplate = { relationships: { documents: { data: [{ type: "document", id: "1234567890" }] }, account: { data: { id: "1234567890", type: "account" } } as const, + "inquiry-template": { data: { id: "template-id" } }, }, }; diff --git a/server/test/utils/persona.test.ts b/server/test/utils/persona.test.ts index 6abba2dd8..f7bbc4775 100644 --- a/server/test/utils/persona.test.ts +++ b/server/test/utils/persona.test.ts @@ -10,6 +10,11 @@ vi.mock("../../utils/panda"); vi.mock("../../utils/pax"); vi.mock("@sentry/node", { spy: true }); +function getFirst(items: T[]): T { + if (items.length !== 1) throw new Error("expected exactly one element"); + return items[0] as T; +} + describe("is missing or null util", () => { const schema = object({ field1: string(), @@ -162,14 +167,14 @@ describe("is missing or null util", () => { describe("evaluateAccount", () => { it("throws when scope is not supported", () => { - expect(() => persona.evaluateAccount({ data: [] }, "invalid" as persona.AccountScope)).toThrow( - "unhandled account scope: invalid", - ); + expect(() => + persona.evaluateAccount({ data: [], links: { next: null } }, "invalid" as persona.AccountScope), + ).toThrow("unhandled account scope: invalid"); }); describe("basic", () => { it("returns panda template when account not found", () => { - const result = persona.evaluateAccount({ data: [] }, "basic"); + const result = persona.evaluateAccount({ data: [], links: { next: null } }, "basic"); expect(result).toBe(persona.PANDA_TEMPLATE); }); @@ -189,7 +194,10 @@ describe("evaluateAccount", () => { it("throws when account exists but is invalid", () => { expect(() => persona.evaluateAccount( - { data: [{ id: "acc-123", type: "account", attributes: { "country-code": 3 } }] }, + { + data: [{ id: "acc-123", attributes: { "reference-id": null, "country-code": 3 } }], + links: { next: null }, + }, "basic", ), ).toThrow(persona.scopeValidationErrors.INVALID_SCOPE_VALIDATION); @@ -198,7 +206,7 @@ describe("evaluateAccount", () => { describe("manteca", () => { it("returns panda template when account not found", () => { - const result = persona.evaluateAccount({ data: [] }, "manteca"); + const result = persona.evaluateAccount({ data: [], links: { next: null } }, "manteca"); expect(result).toBe(persona.PANDA_TEMPLATE); }); @@ -213,13 +221,13 @@ describe("evaluateAccount", () => { expect(() => persona.evaluateAccount( { + links: { next: null }, data: [ { - ...mantecaAccount.data[0], - type: "account" as const, + ...getFirst(mantecaAccount.data), id: "test-account-id", attributes: { - ...mantecaAccount.data[0]?.attributes, + ...getFirst(mantecaAccount.data).attributes, "country-code": "XX", }, }, @@ -237,22 +245,24 @@ describe("evaluateAccount", () => { }); it("returns manteca template when new account has a id class that is not allowed", () => { + const basic = getFirst(basicAccount.data); + const document = getFirst(basic.attributes.fields.documents.value); const result = persona.evaluateAccount( { + links: { next: null }, data: [ { - ...basicAccount.data[0], - type: "account" as const, + ...basic, id: "test-account-id", attributes: { - ...basicAccount.data[0]?.attributes, + ...basic.attributes, fields: { - ...basicAccount.data[0]?.attributes.fields, + ...basic.attributes.fields, documents: { value: [ { value: { - ...basicAccount.data[0]?.attributes.fields.documents.value[0]?.value, + ...document.value, id_class: { value: "invalid" }, }, }, @@ -276,18 +286,19 @@ describe("evaluateAccount", () => { }); it("throws when schema validation fails", () => { + const manteca = getFirst(mantecaAccount.data); expect(() => persona.evaluateAccount( { + links: { next: null }, data: [ { - ...mantecaAccount.data[0], - type: "account" as const, + ...manteca, id: "test-account-id", attributes: { - ...mantecaAccount.data[0]?.attributes, + ...manteca.attributes, fields: { - ...mantecaAccount.data[0]?.attributes.fields, + ...manteca.attributes.fields, tin: { type: "string", value: 123 }, }, }, @@ -302,6 +313,7 @@ describe("evaluateAccount", () => { }); const emptyAccount = { + links: { next: null }, data: [ { type: "account" as const, @@ -451,6 +463,7 @@ const emptyAccount = { }; const basicAccount = { + links: { next: null }, data: [ { type: "account" as const, @@ -665,16 +678,19 @@ const basicAccount = { ], }; +const basicData = getFirst(basicAccount.data); + const mantecaAccount = { + links: { next: null }, data: [ { - ...basicAccount.data[0], + ...basicData, type: "account" as const, id: "test-account-id", attributes: { - ...basicAccount.data[0]?.attributes, + ...basicData.attributes, fields: { - ...basicAccount.data[0]?.attributes.fields, + ...basicData.attributes.fields, tin: { type: "string", value: "12345678", diff --git a/server/utils/persona.ts b/server/utils/persona.ts index 53fb9f572..591cea6c1 100644 --- a/server/utils/persona.ts +++ b/server/utils/persona.ts @@ -4,13 +4,18 @@ import { createHmac, timingSafeEqual } from "node:crypto"; import { array, boolean, + check, flatten, literal, + looseObject, nullable, + nullish, object, picklist, + pipe, safeParse, string, + transform, unknown, ValiError, type BaseIssue, @@ -51,10 +56,87 @@ export async function getInquiry(referenceId: string, templateId: string) { return inquiries[0]; } +export async function getUnknownApprovedInquiry(referenceId: string, templateId?: string) { + const { data: approvedInquiries } = await request( + GetUnknownApprovedInquiryResponse, + `/inquiries?page[size]=1&filter[reference-id]=${referenceId}&filter[status]=approved${templateId ? `&filter[inquiry-template-id]=${templateId}` : ""}`, + ); + return approvedInquiries[0]; +} + export function resumeInquiry(inquiryId: string) { return request(ResumeInquiryResponse, `/inquiries/${inquiryId}/resume`, undefined, "POST"); } +export function redactAccount(accountId: string) { + return request(object({ data: object({ id: string() }) }), `/accounts/${accountId}`, RedactAccount, "PATCH"); +} + +export function updateAccount(accountId: string, fields: InferOutput) { + return request( + object({ data: object({ id: string() }) }), + `/accounts/${accountId}`, + { + data: { + attributes: { + fields, + }, + }, + }, + "PATCH", + ); +} + +export const UpdateAccountFields = object({ + exa_card_tc: boolean(), + rain_e_sign_consent: boolean(), + privacy__policy: boolean(), + account_opening_disclosure: nullable(boolean()), + economic_activity: string(), + annual_salary: string(), + expected_monthly_volume: string(), + accurate_info_confirmation: boolean(), + non_unauthorized_solicitation: boolean(), + non_illegal_activities_2: picklist(["Yes", "No"]), +}); + +const RedactAccount = { + data: { + attributes: { + "country-code": "", + "identification-numbers": "", + "phone-number": "", + "email-address": "", + birthdate: "", + "name-first": "", + "name-middle": "", + "name-last": "", + "address-street-1": "", + "address-street-2": "", + "address-city": "", + "address-subdivision": "", + "address-postal-code": "", + "social-security-number": "", + fields: { + exa_card_tc: "", + rain_e_sign_consent: "", + privacy__policy: "", + account_opening_disclosure: "", + address: { + street_1: "", + street_2: "", + city: "", + subdivision: "", + postal_code: "", + country_code: "", + }, + selfie_photo: "", + documents: [], + }, + }, + }, +}; + export function createInquiry(referenceId: string, templateId: string, redirectURI?: string) { return request(CreateInquiryResponse, "/inquiries", { data: { attributes: { "inquiry-template-id": templateId, "redirect-uri": `${redirectURI ?? appOrigin}/card` } }, @@ -283,7 +365,8 @@ const MantecaAccount = object({ }); const UnknownAccount = object({ - data: array(object({ id: string(), type: literal("account"), attributes: unknown() })), + data: array(looseObject({ id: string(), attributes: looseObject({ "reference-id": nullable(string()) }) })), + links: object({ next: nullable(string()) }), }); const accountScopeSchemas = { @@ -312,6 +395,13 @@ function getUnknownAccount(referenceId: string) { return request(UnknownAccount, `/accounts?page[size]=1&filter[reference-id]=${referenceId}`); } +export function getUnknownAccounts(limit = 1, after?: string, referenceId?: string) { + return request( + UnknownAccount, + `/accounts?page[size]=${limit}${after ? `&page[after]=${after}` : ""}${referenceId ? `&filter[reference-id]=${referenceId}` : ""}`, + ); +} + export async function getPendingInquiryTemplate(referenceId: string, scope: AccountScope): Promise { const unknownAccount = await getUnknownAccount(referenceId); return evaluateAccount(unknownAccount, scope); @@ -383,9 +473,93 @@ export const Inquiry = object({ attributes: object({ status: picklist(["created", "pending", "expired", "failed", "needs_review", "declined", "completed", "approved"]), "reference-id": string(), + "redacted-at": nullable(string()), + }), + relationships: object({ + "inquiry-template": nullable(object({ data: object({ id: string() }) })), + }), +}); + +export const UnknownInquiry = object({ + id: string(), + type: literal("inquiry"), + attributes: object({ + status: literal("approved"), + "reference-id": string(), + "redacted-at": nullable(string()), + fields: unknown(), + }), + relationships: object({ + "inquiry-template": nullable(object({ data: object({ id: string() }) })), }), }); +export const PandaInquiryApproved = object({ + id: string(), + type: literal("inquiry"), + attributes: object({ + status: literal("approved"), + "reference-id": string(), + "redacted-at": nullable(string()), + fields: pipe( + object({ + // common + "input-checkbox": object({ value: boolean() }), // rain e sign consent + "new-screen-input-checkbox": object({ value: boolean() }), // privacy policy + "new-screen-input-checkbox-1": object({ value: boolean() }), // info accurate + "new-screen-input-checkbox-3": object({ value: boolean() }), // unauthorized solicitation + "new-screen-2-2-input-checkbox": nullish(object({ value: nullable(boolean()) })), // exa card tc + + // US + "new-screen-input-checkbox-4": nullish(object({ value: nullable(boolean()) })), // account opening disclosure + "new-screen-input-checkbox-2": nullish(object({ value: nullable(boolean()) })), // exa card tc legacy + + "identification-class": object({ value: string() }), + "identification-number": object({ value: string() }), + "selected-country-code": object({ value: string() }), + "current-government-id": object({ value: object({ id: string() }) }), + + "input-select": object({ value: string() }), // economic activity + + "account-purpose": object({ value: string() }), + "illegal-activites": object({ value: picklist(["Yes", "No"]) }), // cspell:ignore illegal-activites + + // fallback for missing fields + "annual-salary-ranges-us-150-000": nullish(object({ value: nullable(string()) })), + "annual-salary": nullish(object({ value: nullable(string()) })), + + "monthly-purchases-range": nullish(object({ value: nullable(string()) })), + "expected-monthly-volume": nullish(object({ value: nullable(string()) })), + }), + transform((fields) => { + if (!fields["new-screen-2-2-input-checkbox"]?.value && !fields["new-screen-input-checkbox-2"]?.value) { + // eslint-disable-next-line no-console + console.error( + "❌ exa card tc is required, either new-screen-2-2-input-checkbox or new-screen-input-checkbox-2 must be true, setting new-screen-input-checkbox-2 to true", + ); + return { ...fields, "new-screen-input-checkbox-2": { value: true } }; + } + return fields; + }), + check( + (fields) => !!fields["annual-salary"]?.value || !!fields["annual-salary-ranges-us-150-000"]?.value, + "either annual-salary or annual-salary-ranges-us-150-000 must have a value", + ), + check( + (fields) => !!fields["expected-monthly-volume"]?.value || !!fields["monthly-purchases-range"]?.value, + "either expected-monthly-volume or monthly-purchases-range must have a value", + ), + ), + }), + relationships: object({ + "inquiry-template": nullable(object({ data: object({ id: string() }) })), + }), +}); + +const GetUnknownApprovedInquiryResponse = object({ + data: array(UnknownInquiry), +}); + const GetInquiriesResponse = object({ data: array(Inquiry), }); diff --git a/server/utils/validatorHook.ts b/server/utils/validatorHook.ts index 12d74c726..dcb757d24 100644 --- a/server/utils/validatorHook.ts +++ b/server/utils/validatorHook.ts @@ -46,7 +46,7 @@ export default function validatorHook< } /** recursively builds human-readable error messages from valibot validation issues */ -function buildIssueMessages(issues: BaseIssue[], depth = 0, maxDepth = 5): string[] { +export function buildIssueMessages(issues: BaseIssue[], depth = 0, maxDepth = 5): string[] { const messages: string[] = []; for (const issue of issues) { const pathString = issue.path?.map((pathItem) => pathItem.key).join("/") ?? ""; From a2b530378c4a48a8c488c8cfe65ac5fa953e1283 Mon Sep 17 00:00:00 2001 From: mainqueg Date: Fri, 23 Jan 2026 15:42:54 -0300 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=9A=A7=20server:=20kyc=20with=20addre?= =?UTF-8?q?ss=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/script/kyc-migration.ts | 38 +++++++++++++++------ server/utils/persona.ts | 61 +++++++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/server/script/kyc-migration.ts b/server/script/kyc-migration.ts index a50d008d0..331c415a4 100644 --- a/server/script/kyc-migration.ts +++ b/server/script/kyc-migration.ts @@ -169,17 +169,35 @@ function updateAccountFromInquiry(accountId: string, inquiry: InferOutput) { +export function updateAccount(accountId: string, attributes: InferOutput) { return request( object({ data: object({ id: string() }) }), `/accounts/${accountId}`, { data: { - attributes: { - fields, - }, + attributes, }, }, "PATCH", ); } -export const UpdateAccountFields = object({ - exa_card_tc: boolean(), - rain_e_sign_consent: boolean(), - privacy__policy: boolean(), - account_opening_disclosure: nullable(boolean()), - economic_activity: string(), - annual_salary: string(), - expected_monthly_volume: string(), - accurate_info_confirmation: boolean(), - non_unauthorized_solicitation: boolean(), - non_illegal_activities_2: picklist(["Yes", "No"]), +export const UpdateAccountAttributes = object({ + fields: object({ + exa_card_tc: boolean(), + rain_e_sign_consent: boolean(), + privacy__policy: boolean(), + account_opening_disclosure: nullable(boolean()), + economic_activity: string(), + annual_salary: string(), + expected_monthly_volume: string(), + accurate_info_confirmation: boolean(), + non_unauthorized_solicitation: boolean(), + non_illegal_activities_2: picklist(["Yes", "No"]), + address: object({ + value: object({ + street_1: string(), + street_2: nullable(string()), + city: string(), + subdivision: string(), + postal_code: string(), + country_code: string(), + }), + }), + }), + "address-street-1": string(), + "address-street-2": nullable(string()), + "address-city": string(), + "address-subdivision": string(), + "address-postal-code": string(), + "country-code": string(), }); const RedactAccount = { @@ -483,7 +499,7 @@ export const Inquiry = object({ export const UnknownInquiry = object({ id: string(), type: literal("inquiry"), - attributes: object({ + attributes: looseObject({ status: literal("approved"), "reference-id": string(), "redacted-at": nullable(string()), @@ -501,6 +517,12 @@ export const PandaInquiryApproved = object({ status: literal("approved"), "reference-id": string(), "redacted-at": nullable(string()), + // "address-street-1": string(), + // "address-street-2": nullable(string()), + // "address-city": string(), + // "address-subdivision": string(), + // "address-postal-code": string(), + // "address-country-code": string(), fields: pipe( object({ // common @@ -530,6 +552,13 @@ export const PandaInquiryApproved = object({ "monthly-purchases-range": nullish(object({ value: nullable(string()) })), "expected-monthly-volume": nullish(object({ value: nullable(string()) })), + + "address-street-1": object({ value: string() }), + "address-street-2": object({ value: nullable(string()) }), + "address-city": object({ value: string() }), + "address-subdivision": object({ value: string() }), + "address-postal-code": object({ value: string() }), + "address-country-code": object({ value: string() }), }), transform((fields) => { if (!fields["new-screen-2-2-input-checkbox"]?.value && !fields["new-screen-input-checkbox-2"]?.value) { From 5ef0536a16c0b829948d711253ad188a67a69fbe Mon Sep 17 00:00:00 2001 From: mainqueg Date: Mon, 26 Jan 2026 14:11:20 -0300 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=9A=A7=20server:=20filter=20accounts?= =?UTF-8?q?=20from=20csv=20entries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/script/kyc-migration.ts | 103 ++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/server/script/kyc-migration.ts b/server/script/kyc-migration.ts index 331c415a4..17a531244 100644 --- a/server/script/kyc-migration.ts +++ b/server/script/kyc-migration.ts @@ -1,10 +1,41 @@ import createDebug from "debug"; +import { createReadStream } from "node:fs"; +import { createInterface } from "node:readline"; import { inspect } from "node:util"; import { safeParse, ValiError, type InferOutput } from "valibot"; import * as persona from "../utils/persona"; import { buildIssueMessages } from "../utils/validatorHook"; +type FilterMode = "none" | "only-failed" | "skip-completed"; + +type CsvEntry = { + accountId: string; + referenceId: string; +}; + +async function loadEntriesFromCsv(csvPath: string): Promise { + const entries: CsvEntry[] = []; + // eslint-disable-next-line security/detect-non-literal-fs-filename + const fileStream = createReadStream(csvPath); + const rl = createInterface({ input: fileStream, crlfDelay: Number.POSITIVE_INFINITY }); + + let isHeader = true; + for await (const line of rl) { + if (isHeader) { + isHeader = false; + continue; + } + // parse csv line: "referenceId","accountId","status" + const match = /^"([^"]*)","([^"]*)"/.exec(line); + if (match?.[1] && match[2]) { + entries.push({ referenceId: match[1], accountId: match[2] }); + } + } + + return entries; +} + const BATCH_SIZE = 10; const debug = createDebug("migration:debug"); @@ -17,6 +48,8 @@ let all = false; let onlyLogs = false; let initialNext: string | undefined; let onlyPandaTemplates = false; +let filterMode: FilterMode = "none"; +let filterCsvPath: string | undefined; const options = process.argv.slice(2); for (const option of options) { @@ -37,6 +70,14 @@ for (const option of options) { case option.startsWith("--only-panda-templates"): onlyPandaTemplates = true; break; + case option.startsWith("--skip-completed="): + filterMode = "skip-completed"; + filterCsvPath = option.split("=")[1]; + break; + case option.startsWith("--only-failed="): + filterMode = "only-failed"; + filterCsvPath = option.split("=")[1]; + break; default: unexpected(`❌ unknown option: ${option}`); throw new Error(`unknown option: ${option}`); @@ -60,8 +101,15 @@ let failedAccounts = 0; let inquirySchemaErrors = 0; let noReferenceIdAccounts = 0; let totalAccounts = 0; +let skippedByFilter = 0; async function main() { + if (filterMode === "only-failed" && filterCsvPath) { + await processOnlyFailed(filterCsvPath); + printSummary(); + return; + } + if (all) { log("🔍 Processing all accounts"); } else if (reference) { @@ -71,6 +119,14 @@ async function main() { throw new Error("missing --reference-id= or --all"); } + let skipSet: Set | undefined; + if (filterMode === "skip-completed" && filterCsvPath) { + const entries = await loadEntriesFromCsv(filterCsvPath); + skipSet = new Set(entries.map((entry) => entry.referenceId)); + log(`📋 Loaded ${skipSet.size} reference IDs to skip from ${filterCsvPath}`); + log("🔄 Mode: skip-completed (will skip accounts in CSV)"); + } + let next = initialNext; let batch = 0; let retries = 0; @@ -100,7 +156,16 @@ async function main() { warn(`Account ${account.id} has no reference id`); continue; } - await processAccount(account.id, account.attributes["reference-id"]); + + const accountReferenceId = account.attributes["reference-id"]; + + if (skipSet?.has(accountReferenceId)) { + skippedByFilter++; + debug(`⏭️ Skipping completed account ${accountReferenceId}/${account.id}`); + continue; + } + + await processAccount(account.id, accountReferenceId); } catch (error: unknown) { unexpected( `❌ Failed to process batch ${batch}, next: ${next ?? "undefined"}, account: ${account.id}/${account.attributes["reference-id"]} due to: ${inspect(error, { depth: null, colors: true })}`, @@ -125,8 +190,44 @@ async function main() { } } + printSummary(); +} + +async function processOnlyFailed(csvPath: string) { + const entries = await loadEntriesFromCsv(csvPath); + log(`📋 Loaded ${entries.length} failed accounts from ${csvPath}`); + log("🔄 Mode: only-failed (processing accounts directly from CSV)"); + + totalAccounts = entries.length; + + for (let index = 0; index < entries.length; index++) { + const entry = entries[index]; + if (!entry) continue; + + const { referenceId, accountId } = entry; + + if (index % BATCH_SIZE === 0) { + log(`\n ----- Processing batch ${Math.floor(index / BATCH_SIZE)} (${index}/${entries.length}) -----`); + } + + try { + await processAccount(accountId, referenceId); + } catch (error: unknown) { + unexpected( + `❌ Failed to process account ${accountId}/${referenceId} due to: ${inspect(error, { depth: null, colors: true })}`, + ); + failedAccounts++; + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } +} + +function printSummary() { log(`\n ----- Migration summary -----`); log(`🔍 Total accounts processed: ${totalAccounts}`); + if (skippedByFilter > 0) { + log(`⏭️ Skipped by filter: ${skippedByFilter}`); + } log(`🔍 Redacted inquiries: ${redactedInquiries}`); log(`🔍 No approved inquiry accounts, redaction needed: ${noApprovedInquiryAccounts}`); log(` ---------------------------------`);