diff --git a/packages/app-store/googlecalendar/lib/CalendarAuth.ts b/packages/app-store/googlecalendar/lib/CalendarAuth.ts index 6ead261afb5a02..3a4e393c47ae15 100644 --- a/packages/app-store/googlecalendar/lib/CalendarAuth.ts +++ b/packages/app-store/googlecalendar/lib/CalendarAuth.ts @@ -1,11 +1,8 @@ -import { calendar_v3 } from "@googleapis/calendar"; -import { OAuth2Client, JWT } from "googleapis-common"; - import { triggerDelegationCredentialErrorWebhook } from "@calcom/features/webhooks/lib/triggerDelegationCredentialErrorWebhook"; import { CalendarAppDelegationCredentialClientIdNotAuthorizedError, - CalendarAppDelegationCredentialInvalidGrantError, CalendarAppDelegationCredentialError, + CalendarAppDelegationCredentialInvalidGrantError, } from "@calcom/lib/CalendarAppError"; import { APP_CREDENTIAL_SHARING_ENABLED, @@ -16,7 +13,9 @@ import { import logger from "@calcom/lib/logger"; import type { Prisma } from "@calcom/prisma/client"; import type { CredentialForCalendarServiceWithEmail } from "@calcom/types/Credential"; - +import { calendar_v3 } from "@googleapis/calendar"; +import type { GaxiosError, RetryConfig } from "googleapis-common"; +import { JWT, OAuth2Client } from "googleapis-common"; import { invalidateCredential } from "../../_utils/invalidateCredential"; import { OAuthManager } from "../../_utils/oauth/OAuthManager"; import { oAuthManagerHelper } from "../../_utils/oauth/oAuthManagerHelper"; @@ -25,7 +24,59 @@ import { metadata } from "../_metadata"; import { getGoogleAppKeys } from "./getGoogleAppKeys"; type DelegatedTo = NonNullable; -const log = logger.getSubLogger({ prefix: ["app-store/googlecalendar/lib/CalendarAuth"] }); +const log: typeof logger = logger.getSubLogger({ prefix: ["app-store/googlecalendar/lib/CalendarAuth"] }); + +const GOOGLE_CALENDAR_RETRY_CONFIG: RetryConfig = { + retry: 3, + noResponseRetries: 2, + httpMethodsToRetry: ["GET", "HEAD", "PUT", "OPTIONS", "DELETE", "PATCH", "POST"], + statusCodesToRetry: [ + [100, 199], + [403, 403], + [429, 429], + [500, 599], + ], + shouldRetry: (error: GaxiosError): boolean => shouldRetryGoogleCalendarRequest(error), +}; + +function hasRetriesRemaining(retryConfig: RetryConfig): boolean { + const currentRetryAttempt = retryConfig.currentRetryAttempt ?? 0; + const maxRetries = retryConfig.retry ?? 0; + return currentRetryAttempt < maxRetries; +} + +function isRetryableStatus(status: number, retryConfig: RetryConfig): boolean { + return retryConfig.statusCodesToRetry?.some(([min, max]) => status >= min && status <= max) ?? false; +} + +function isGoogleRateLimitError(error: GaxiosError): boolean { + const reason = (error.response?.data as { error?: { errors?: Array<{ reason?: string }> } } | undefined) + ?.error?.errors?.[0]?.reason; + + return reason === "rateLimitExceeded" || reason === "userRateLimitExceeded"; +} + +function shouldRetryGoogleCalendarRequest(error: GaxiosError): boolean { + const retryConfig = error.config.retryConfig; + + if (!retryConfig || error.name === "AbortError") return false; + if (!hasRetriesRemaining(retryConfig)) return false; + + if (!error.response) { + return (retryConfig.currentRetryAttempt ?? 0) < (retryConfig.noResponseRetries ?? 0); + } + + const method = error.config.method?.toUpperCase(); + if (!method || !retryConfig.httpMethodsToRetry?.includes(method)) return false; + + if (!isRetryableStatus(error.response.status, retryConfig)) return false; + + if (error.response.status === 403) { + return isGoogleRateLimitError(error); + } + + return true; +} class MyGoogleOAuth2Client extends OAuth2Client { constructor(client_id: string, client_secret: string, redirect_uri: string) { @@ -216,7 +267,7 @@ export class CalendarAuth { statusText: result.statusText, }); }, - isTokenObjectUnusable: async function (response) { + isTokenObjectUnusable: async (response) => { // TODO: Confirm that if this logic should go to isAccessTokenUnusable if (!response.ok || (response.status < 200 && response.status >= 300)) { const responseBody = await response.json(); @@ -303,6 +354,7 @@ export class CalendarAuth { return new calendar_v3.Calendar({ auth: googleAuthClient, + retryConfig: GOOGLE_CALENDAR_RETRY_CONFIG, }); } } diff --git a/packages/app-store/googlecalendar/lib/__tests__/CalendarService.auth.test.ts b/packages/app-store/googlecalendar/lib/__tests__/CalendarService.auth.test.ts index 4cf07433ef9991..e34554150fc6bd 100644 --- a/packages/app-store/googlecalendar/lib/__tests__/CalendarService.auth.test.ts +++ b/packages/app-store/googlecalendar/lib/__tests__/CalendarService.auth.test.ts @@ -2,42 +2,41 @@ import prismock from "@calcom/testing/lib/__mocks__/prisma"; import "../__mocks__/features.repository"; import "../__mocks__/getGoogleAppKeys"; import { - setCredentialsMock, + adminMock, calendarListMock, + calendarMock, getLastCreatedJWT, getLastCreatedOAuth2Client, - setLastCreatedJWT, - setLastCreatedOAuth2Client, - calendarMock, - adminMock, MOCK_JWT_TOKEN, MOCK_OAUTH2_TOKEN, + setCredentialsMock, + setLastCreatedJWT, + setLastCreatedOAuth2Client, } from "../__mocks__/googleapis"; - -import { expect, test, beforeEach, vi, describe } from "vitest"; +import { beforeEach, describe, expect, test, vi } from "vitest"; import "vitest-fetch-mock"; import type { CredentialForCalendarServiceWithEmail } from "@calcom/types/Credential"; - +import type { GaxiosError, RetryConfig } from "googleapis-common"; import BuildCalendarService from "../CalendarService"; import { - createMockJWTInstance, + createCredentialForCalendarService, createInMemoryDelegationCredentialForCalendarService as createInMemoryDelegationCredentialForBuildCalendarService, + createMockJWTInstance, defaultDelegatedCredential, - createCredentialForCalendarService, } from "./utils"; -function expectJWTInstanceToBeCreated() { +function expectJWTInstanceToBeCreated(): void { expect(getLastCreatedJWT()).toBeDefined(); expect(setCredentialsMock).not.toHaveBeenCalled(); } -function expectOAuth2InstanceToBeCreated() { +function expectOAuth2InstanceToBeCreated(): void { expect(setCredentialsMock).toHaveBeenCalled(); expect(getLastCreatedJWT()).toBeNull(); } -function mockSuccessfulCalendarListFetch() { +function mockSuccessfulCalendarListFetch(): void { calendarListMock.mockImplementation(() => { return { data: { items: [] }, @@ -45,6 +44,49 @@ function mockSuccessfulCalendarListFetch() { }); } +function getCalendarRetryConfig(): RetryConfig { + const config = calendarMock.calendar_v3.Calendar.mock.calls.at(-1)?.[0]?.retryConfig as + | RetryConfig + | undefined; + expect(config).toBeDefined(); + return config as RetryConfig; +} + +function createGoogleApiError({ + method, + status, + reason, +}: { + method: "PATCH" | "POST" | "GET"; + status: number; + reason?: string; +}): GaxiosError { + const retryConfig = getCalendarRetryConfig(); + let data: { error: { errors: Array<{ reason: string }> } } | undefined; + + if (reason) { + data = { + error: { + errors: [{ reason }], + }, + }; + } + + return Object.assign(new Error("Google Calendar request failed"), { + config: { + method, + retryConfig: { + ...retryConfig, + currentRetryAttempt: 0, + }, + }, + response: { + status, + data, + }, + }) as GaxiosError; +} + beforeEach(() => { vi.clearAllMocks(); setCredentialsMock.mockClear(); @@ -56,12 +98,12 @@ beforeEach(() => { createMockJWTInstance({}); }); -async function expectNoCredentialsInDb() { +async function expectNoCredentialsInDb(): Promise { const credentials = await prismock.credential.findMany({}); expect(credentials).toHaveLength(0); } -async function expectCredentialsInDb(credentials: CredentialForCalendarServiceWithEmail[]) { +async function expectCredentialsInDb(credentials: CredentialForCalendarServiceWithEmail[]): Promise { const credentialsInDb = await prismock.credential.findMany({}); expect(credentialsInDb.length).toBe(credentials.length); expect(credentialsInDb).toEqual(expect.arrayContaining(credentials)); @@ -193,9 +235,11 @@ describe("GoogleCalendarService credential handling", () => { expectOAuth2InstanceToBeCreated(); - expect(calendarMock.calendar_v3.Calendar).toHaveBeenCalledWith({ - auth: getLastCreatedOAuth2Client(), - }); + expect(calendarMock.calendar_v3.Calendar).toHaveBeenCalledWith( + expect.objectContaining({ + auth: getLastCreatedOAuth2Client(), + }) + ); await expectCredentialsInDb([ expect.objectContaining({ id: regularCredential.id, @@ -203,6 +247,73 @@ describe("GoogleCalendarService credential handling", () => { }), ]); }); + + test("instantiates the calendar client with retry config for Google Calendar write requests", async () => { + const regularCredential = await createCredentialForCalendarService(); + mockSuccessfulCalendarListFetch(); + const calendarService = BuildCalendarService(regularCredential); + + await calendarService.listCalendars(); + + expect(calendarMock.calendar_v3.Calendar).toHaveBeenCalledWith( + expect.objectContaining({ + retryConfig: expect.objectContaining({ + retry: 3, + noResponseRetries: 2, + httpMethodsToRetry: expect.arrayContaining(["PATCH", "POST"]), + statusCodesToRetry: expect.arrayContaining([ + [403, 403], + [429, 429], + [500, 599], + ]), + }), + }) + ); + }); + + test("retries Google 403 rate limit errors for PATCH requests", async () => { + const regularCredential = await createCredentialForCalendarService(); + mockSuccessfulCalendarListFetch(); + const calendarService = BuildCalendarService(regularCredential); + + await calendarService.listCalendars(); + + const retryConfig = getCalendarRetryConfig(); + const shouldRetry = retryConfig.shouldRetry; + + expect(shouldRetry).toBeDefined(); + expect( + shouldRetry?.( + createGoogleApiError({ + method: "PATCH", + status: 403, + reason: "rateLimitExceeded", + }) + ) + ).toBe(true); + }); + + test("does not retry non-rate-limit 403 errors for PATCH requests", async () => { + const regularCredential = await createCredentialForCalendarService(); + mockSuccessfulCalendarListFetch(); + const calendarService = BuildCalendarService(regularCredential); + + await calendarService.listCalendars(); + + const retryConfig = getCalendarRetryConfig(); + const shouldRetry = retryConfig.shouldRetry; + + expect(shouldRetry).toBeDefined(); + expect( + shouldRetry?.( + createGoogleApiError({ + method: "PATCH", + status: 403, + reason: "forbidden", + }) + ) + ).toBe(false); + }); }); describe("Delegation Credential Error handling", () => {