feat(webhook): add a retry webhook strategy

migration/1768855049768-AddNextTryDate.js

@@ -0,0 +1,26 @@
+/**
+ * @typedef {import('typeorm').MigrationInterface} MigrationInterface
+ * @typedef {import('typeorm').QueryRunner} QueryRunner
+ */
+
+/**
+ * @class
+ * @implements {MigrationInterface}
+ */
+module.exports = class AddNextTryDate1768855049768 {
+    name = 'AddNextTryDate1768855049768'
+
+    /**
+     * @param {QueryRunner} queryRunner
+     */
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "webhook" ADD "nextTryDate" datetime2`);
+    }
+
+    /**
+     * @param {QueryRunner} queryRunner
+     */
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "webhook" DROP COLUMN "nextTryDate"`);
+    }
+}

src/subdomains/generic/user/services/webhook/webhook-notification.service.ts

@@ -4,9 +4,10 @@ import { DfxLogger } from 'src/shared/services/dfx-logger';
 import { HttpService } from 'src/shared/services/http.service';
 import { Process } from 'src/shared/services/process.service';
 import { DfxCron } from 'src/shared/utils/cron';
+import { Util } from 'src/shared/utils/util';
 import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums';
 import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service';
-import { IsNull } from 'typeorm';
+import { IsNull, LessThanOrEqual } from 'typeorm';
 import { KycWebhookData } from './dto/kyc-webhook.dto';
 import { PaymentWebhookData } from './dto/payment-webhook.dto';
 import { WebhookDto } from './dto/webhook.dto';
@@ -29,56 +30,93 @@ export class WebhookNotificationService {
   }
 
   async sendOpenWebhooks(): Promise<void> {
-    const entities = await this.webhookRepo.find({ where: { lastTryDate: IsNull() } });
+    const now = new Date();
 
-    if (entities.length > 0) this.logger.verbose(`Sending ${entities.length} 'webhooks`);
+    const entities = await this.webhookRepo.find({
+      where: [{ nextTryDate: LessThanOrEqual(now) }, { nextTryDate: IsNull(), lastTryDate: IsNull() }],
+      relations: ['wallet', 'user', 'userData'],
+    });
+
+    if (entities.length > 0) this.logger.verbose(`Sending ${entities.length} webhooks`);
 
     for (const entity of entities) {
-      try {
-        const result = await this.triggerWebhook(entity);
-        await this.webhookRepo.update(...entity.sentWebhook(result));
-      } catch (e) {
-        this.logger.error(`Failed to send webhook ${entity.id}:`, e);
-      }
+      await this.handleSingleWebhook(entity);
     }
   }
 
-  // --- HELPER METHODS --- //
+  private async handleSingleWebhook(webhook: Webhook): Promise<void> {
+    const now = new Date();
 
-  async triggerWebhook<T extends PaymentWebhookData | KycWebhookData>(webhook: Webhook): Promise<string | undefined> {
     try {
-      if (!webhook.wallet.apiUrl)
-        throw new Error(`API URL for wallet ${webhook.wallet.name} not available anymore in webhook ${webhook.id}`);
-
-      const webhookDto: WebhookDto<T> = {
-        accountId: webhook.userData.id,
-        id: webhook.user?.address,
-        type: webhook.type,
-        data: JSON.parse(webhook.data),
-        reason: webhook.reason,
-      };
-
-      await this.http.post(webhook.wallet.apiUrl, webhookDto, {
-        headers: { 'x-api-key': webhook.wallet.apiKey },
-        retryDelay: 5000,
-        tryCount: 3,
-      });
+      const result = await this.triggerWebhook(webhook);
+
+      await this.webhookRepo.update(
+        { id: webhook.id },
+        {
+          ...webhook.sentWebhook(result),
+          lastTryDate: now,
+          nextTryDate: null,
+        },
+      );
     } catch (error) {
-      const id = webhook.id ?? webhook.identifier;
-      const errMessage = `Exception during webhook for user data ${webhook.userData.id} and wallet ${webhook.wallet.name} (webhook ${id}):`;
+      const nextTryDate = this.getNextTryDateOrNull(webhook);
 
-      this.logger.error(errMessage, error);
+      if (!nextTryDate) {
+        await this.sendManualCheckMail(webhook, error);
+      }
 
-      await this.notificationService.sendMail({
-        type: MailType.ERROR_MONITORING,
-        context: MailContext.WEBHOOK,
-        input: {
-          subject: `Webhook ${id} failed`,
-          errors: [errMessage, error],
+      await this.webhookRepo.update(
+        { id: webhook.id },
+        {
+          lastTryDate: now,
+          nextTryDate,
         },
-      });
-
-      return error.message;
+      );
     }
   }
+
+  // --- HELPER METHODS --- //
+
+  async triggerWebhook<T extends PaymentWebhookData | KycWebhookData>(webhook: Webhook): Promise<string | undefined> {
+    if (!webhook.wallet.apiUrl)
+      throw new Error(`API URL for wallet ${webhook.wallet.name} not available anymore in webhook ${webhook.id}`);
+
+    const webhookDto: WebhookDto<T> = {
+      accountId: webhook.userData.id,
+      id: webhook.user?.address,
+      type: webhook.type,
+      data: JSON.parse(webhook.data),
+      reason: webhook.reason,
+    };
+
+    await this.http.post(webhook.wallet.apiUrl, webhookDto, {
+      headers: { 'x-api-key': webhook.wallet.apiKey },
+      retryDelay: 5000,
+      tryCount: 3,
+    });
+
+    return undefined;
+  }
+
+  private async sendManualCheckMail(webhook: Webhook, error: any): Promise<void> {
+    const id = webhook.id ?? webhook.identifier;
+    const errMessage = `Webhook for user data ${webhook.userData.id} and wallet ${webhook.wallet.name} (webhook ${id}) failed for more than 24h.`;
+
+    await this.notificationService.sendMail({
+      type: MailType.ERROR_MONITORING,
+      context: MailContext.WEBHOOK,
+      input: {
+        subject: `Webhook ${id} requires manual check`,
+        errors: [errMessage, error],
+      },
+    });
+  }
+
+  private getNextTryDateOrNull(webhook: Webhook): Date | null {
+    const ageMinutes = (new Date().getTime() - webhook.created.getTime()) / 60000;
+    if (ageMinutes >= 24 * 60) return null;
+
+    const delayMinutes = ageMinutes < 60 ? 5 : 60;
+    return Util.minutesAfter(delayMinutes);
+  }
 }

src/subdomains/generic/user/services/webhook/webhook.entity.ts

@@ -22,6 +22,9 @@ export class Webhook extends IEntity {
   @Column({ type: 'datetime2', nullable: true })
   lastTryDate?: Date;
 
+  @Column({ type: 'datetime2', nullable: true })
+  nextTryDate?: Date;
+
   @Column({ length: 'MAX', nullable: true })
   error?: string;