From 340d9ad4c1504575fa496cd70918a2308d754c2b Mon Sep 17 00:00:00 2001 From: Uche Solomon <100881766+Devsol-01@users.noreply.github.com> Date: Mon, 30 Mar 2026 07:38:43 +0100 Subject: [PATCH] feat: add A/B testing framework for savings products (closes #483) --- ...200000000-CreateSavingsExperimentTables.ts | 98 ++++++ .../modules/admin/admin-savings.controller.ts | 69 +++- .../savings-experiment-assignment.entity.ts | 40 +++ .../entities/savings-experiment.entity.ts | 62 ++++ .../modules/savings/experiments.service.ts | 294 ++++++++++++++++++ backend/src/modules/savings/savings.module.ts | 14 +- 6 files changed, 574 insertions(+), 3 deletions(-) create mode 100644 backend/src/migrations/1791200000000-CreateSavingsExperimentTables.ts create mode 100644 backend/src/modules/savings/entities/savings-experiment-assignment.entity.ts create mode 100644 backend/src/modules/savings/entities/savings-experiment.entity.ts create mode 100644 backend/src/modules/savings/experiments.service.ts diff --git a/backend/src/migrations/1791200000000-CreateSavingsExperimentTables.ts b/backend/src/migrations/1791200000000-CreateSavingsExperimentTables.ts new file mode 100644 index 000000000..d9b3bbe64 --- /dev/null +++ b/backend/src/migrations/1791200000000-CreateSavingsExperimentTables.ts @@ -0,0 +1,98 @@ +import { MigrationInterface, QueryRunner, Table, TableIndex } from 'typeorm'; + +export class CreateSavingsExperimentTables1791200000000 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: 'savings_experiments', + columns: [ + { + name: 'id', + type: 'uuid', + isPrimary: true, + isGenerated: true, + generationStrategy: 'uuid', + }, + { name: 'key', type: 'varchar', isUnique: true }, + { name: 'name', type: 'varchar' }, + { name: 'description', type: 'text', isNullable: true }, + { name: 'productId', type: 'uuid', isNullable: true }, + { name: 'variants', type: 'jsonb' }, + { name: 'configuration', type: 'jsonb', isNullable: true }, + { name: 'status', type: 'varchar', default: "'DRAFT'" }, + { name: 'minSampleSize', type: 'int', default: 100 }, + { + name: 'confidenceLevel', + type: 'decimal', + precision: 4, + scale: 2, + default: 0.95, + }, + { name: 'startedAt', type: 'timestamp', isNullable: true }, + { name: 'endedAt', type: 'timestamp', isNullable: true }, + { name: 'createdAt', type: 'timestamp', default: 'now()' }, + { name: 'updatedAt', type: 'timestamp', default: 'now()' }, + ], + }), + ); + + await queryRunner.createTable( + new Table({ + name: 'savings_experiment_assignments', + columns: [ + { + name: 'id', + type: 'uuid', + isPrimary: true, + isGenerated: true, + generationStrategy: 'uuid', + }, + { name: 'experimentId', type: 'uuid' }, + { name: 'userId', type: 'uuid' }, + { name: 'variantKey', type: 'varchar' }, + { name: 'convertedAt', type: 'timestamp', isNullable: true }, + { + name: 'conversionValue', + type: 'decimal', + precision: 14, + scale: 2, + default: 0, + }, + { name: 'metadata', type: 'jsonb', isNullable: true }, + { name: 'createdAt', type: 'timestamp', default: 'now()' }, + { name: 'updatedAt', type: 'timestamp', default: 'now()' }, + ], + }), + ); + + await queryRunner.createIndex( + 'savings_experiment_assignments', + new TableIndex({ + name: 'IDX_savings_experiment_user_unique', + columnNames: ['experimentId', 'userId'], + isUnique: true, + }), + ); + + await queryRunner.createIndex( + 'savings_experiment_assignments', + new TableIndex({ + name: 'IDX_savings_experiment_variant', + columnNames: ['experimentId', 'variantKey'], + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropIndex( + 'savings_experiment_assignments', + 'IDX_savings_experiment_variant', + ); + await queryRunner.dropIndex( + 'savings_experiment_assignments', + 'IDX_savings_experiment_user_unique', + ); + await queryRunner.dropTable('savings_experiment_assignments'); + await queryRunner.dropTable('savings_experiments'); + } +} diff --git a/backend/src/modules/admin/admin-savings.controller.ts b/backend/src/modules/admin/admin-savings.controller.ts index d0bab6bc4..c3d6f90c5 100644 --- a/backend/src/modules/admin/admin-savings.controller.ts +++ b/backend/src/modules/admin/admin-savings.controller.ts @@ -23,6 +23,7 @@ import { JwtAuthGuard } from '../../auth/guards/jwt-auth.guard'; import { RolesGuard } from '../../common/guards/roles.guard'; import { Roles } from '../../common/decorators/roles.decorator'; import { Role } from '../../common/enums/role.enum'; +import { ExperimentsService } from '../savings/experiments.service'; @ApiTags('admin/savings') @Controller('admin/savings') @@ -30,7 +31,10 @@ import { Role } from '../../common/enums/role.enum'; @Roles(Role.ADMIN) @ApiBearerAuth() export class AdminSavingsController { - constructor(private readonly savingsService: SavingsService) {} + constructor( + private readonly savingsService: SavingsService, + private readonly experimentsService: ExperimentsService, + ) {} @Post('products') @HttpCode(HttpStatus.CREATED) @@ -63,4 +67,67 @@ export class AdminSavingsController { ): Promise { return await this.savingsService.updateProduct(id, dto); } + + @Post('experiments') + @ApiOperation({ summary: 'Create a savings product experiment (admin)' }) + @ApiResponse({ status: 201, description: 'Experiment created' }) + async createExperiment( + @Body() + body: { + key: string; + name: string; + description?: string; + productId?: string; + variants: Array<{ + key: string; + weight: number; + config: Record; + }>; + configuration?: Record; + minSampleSize?: number; + confidenceLevel?: number; + status?: 'DRAFT' | 'ACTIVE' | 'PAUSED' | 'COMPLETED'; + }, + ) { + return await this.experimentsService.createExperiment(body); + } + + @Post('experiments/:id/assignments') + @ApiOperation({ summary: 'Assign a user to an experiment variant (admin)' }) + @ApiResponse({ status: 200, description: 'User assigned to a variant' }) + async assignExperimentUser( + @Param('id') id: string, + @Body() body: { userId: string }, + ) { + return await this.experimentsService.assignUser(id, body.userId); + } + + @Post('experiments/:id/conversions') + @ApiOperation({ summary: 'Track an experiment conversion event (admin)' }) + @ApiResponse({ status: 200, description: 'Conversion tracked' }) + async trackExperimentConversion( + @Param('id') id: string, + @Body() + body: { + userId: string; + value?: number; + metadata?: Record; + }, + ) { + return await this.experimentsService.trackConversion( + id, + body.userId, + body.value ?? 1, + body.metadata, + ); + } + + @Post('experiments/:id/dashboard') + @ApiOperation({ summary: 'Get experiment dashboard statistics (admin)' }) + @ApiResponse({ status: 200, description: 'Experiment dashboard' }) + async getExperimentDashboard( + @Param('id') id: string, + ): Promise> { + return await this.experimentsService.getDashboard(id); + } } diff --git a/backend/src/modules/savings/entities/savings-experiment-assignment.entity.ts b/backend/src/modules/savings/entities/savings-experiment-assignment.entity.ts new file mode 100644 index 000000000..4be6b14f0 --- /dev/null +++ b/backend/src/modules/savings/entities/savings-experiment-assignment.entity.ts @@ -0,0 +1,40 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + UpdateDateColumn, + Index, +} from 'typeorm'; + +@Entity('savings_experiment_assignments') +@Index(['experimentId', 'userId'], { unique: true }) +@Index(['experimentId', 'variantKey']) +export class SavingsExperimentAssignment { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column('uuid') + experimentId: string; + + @Column('uuid') + userId: string; + + @Column() + variantKey: string; + + @Column({ type: 'timestamp', nullable: true }) + convertedAt: Date | null; + + @Column('decimal', { precision: 14, scale: 2, default: 0 }) + conversionValue: number; + + @Column({ type: 'jsonb', nullable: true }) + metadata: Record | null; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/backend/src/modules/savings/entities/savings-experiment.entity.ts b/backend/src/modules/savings/entities/savings-experiment.entity.ts new file mode 100644 index 000000000..bf676b011 --- /dev/null +++ b/backend/src/modules/savings/entities/savings-experiment.entity.ts @@ -0,0 +1,62 @@ +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + UpdateDateColumn, + Index, +} from 'typeorm'; + +export type ExperimentStatus = 'DRAFT' | 'ACTIVE' | 'PAUSED' | 'COMPLETED'; + +export interface ExperimentVariant { + key: string; + weight: number; + config: Record; +} + +@Entity('savings_experiments') +@Index(['status', 'createdAt']) +export class SavingsExperiment { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ unique: true }) + key: string; + + @Column() + name: string; + + @Column({ type: 'text', nullable: true }) + description: string | null; + + @Column({ type: 'uuid', nullable: true }) + productId: string | null; + + @Column({ type: 'jsonb' }) + variants: ExperimentVariant[]; + + @Column({ type: 'jsonb', nullable: true }) + configuration: Record | null; + + @Column({ type: 'varchar', default: 'DRAFT' }) + status: ExperimentStatus; + + @Column({ type: 'int', default: 100 }) + minSampleSize: number; + + @Column({ type: 'decimal', precision: 4, scale: 2, default: 0.95 }) + confidenceLevel: number; + + @Column({ type: 'timestamp', nullable: true }) + startedAt: Date | null; + + @Column({ type: 'timestamp', nullable: true }) + endedAt: Date | null; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/backend/src/modules/savings/experiments.service.ts b/backend/src/modules/savings/experiments.service.ts new file mode 100644 index 000000000..58645ce72 --- /dev/null +++ b/backend/src/modules/savings/experiments.service.ts @@ -0,0 +1,294 @@ +import { + BadRequestException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { createHash } from 'crypto'; +import { Repository } from 'typeorm'; +import { + ExperimentVariant, + SavingsExperiment, +} from './entities/savings-experiment.entity'; +import { SavingsExperimentAssignment } from './entities/savings-experiment-assignment.entity'; + +interface VariantDashboard { + key: string; + assignments: number; + conversions: number; + conversionRate: number; + totalConversionValue: number; +} + +@Injectable() +export class ExperimentsService { + constructor( + @InjectRepository(SavingsExperiment) + private readonly experimentRepository: Repository, + @InjectRepository(SavingsExperimentAssignment) + private readonly assignmentRepository: Repository, + ) {} + + async createExperiment(input: { + key: string; + name: string; + description?: string; + productId?: string; + variants: ExperimentVariant[]; + configuration?: Record; + minSampleSize?: number; + confidenceLevel?: number; + status?: SavingsExperiment['status']; + }): Promise { + this.validateVariants(input.variants); + + const experiment = this.experimentRepository.create({ + key: input.key, + name: input.name, + description: input.description ?? null, + productId: input.productId ?? null, + variants: input.variants, + configuration: input.configuration ?? null, + minSampleSize: input.minSampleSize ?? 100, + confidenceLevel: input.confidenceLevel ?? 0.95, + status: input.status ?? 'DRAFT', + startedAt: input.status === 'ACTIVE' ? new Date() : null, + }); + + return await this.experimentRepository.save(experiment); + } + + async listExperiments(): Promise { + return await this.experimentRepository.find({ + order: { createdAt: 'DESC' }, + }); + } + + async assignUser(experimentId: string, userId: string) { + const experiment = await this.findExperiment(experimentId); + if (experiment.status !== 'ACTIVE') { + throw new BadRequestException('Experiment is not active'); + } + + const existing = await this.assignmentRepository.findOneBy({ + experimentId, + userId, + }); + if (existing) { + return existing; + } + + const variantKey = this.pickVariant(experiment, userId); + const assignment = this.assignmentRepository.create({ + experimentId, + userId, + variantKey, + convertedAt: null, + conversionValue: 0, + metadata: null, + }); + + return await this.assignmentRepository.save(assignment); + } + + async trackConversion( + experimentId: string, + userId: string, + value = 1, + metadata?: Record, + ): Promise { + const assignment = await this.assignmentRepository.findOneBy({ + experimentId, + userId, + }); + + if (!assignment) { + throw new NotFoundException('Experiment assignment not found'); + } + + assignment.convertedAt = assignment.convertedAt ?? new Date(); + assignment.conversionValue = value; + assignment.metadata = metadata ?? assignment.metadata; + + return await this.assignmentRepository.save(assignment); + } + + async getDashboard(experimentId: string) { + const experiment = await this.findExperiment(experimentId); + const assignments = await this.assignmentRepository.find({ + where: { experimentId }, + }); + + const variants = experiment.variants.map((variant) => + this.buildVariantDashboard(variant, assignments), + ); + + const rankedVariants = [...variants].sort( + (left, right) => right.conversionRate - left.conversionRate, + ); + const [bestVariant, baselineVariant] = rankedVariants; + const significance = + bestVariant && baselineVariant && bestVariant.key !== baselineVariant.key + ? this.calculateSignificance( + baselineVariant, + bestVariant, + Number(experiment.confidenceLevel ?? 0.95), + ) + : null; + + return { + experimentId: experiment.id, + key: experiment.key, + name: experiment.name, + status: experiment.status, + minSampleSize: experiment.minSampleSize, + confidenceLevel: Number(experiment.confidenceLevel ?? 0.95), + variants, + significance, + }; + } + + private async findExperiment( + experimentId: string, + ): Promise { + const experiment = await this.experimentRepository.findOneBy({ + id: experimentId, + }); + if (!experiment) { + throw new NotFoundException(`Experiment ${experimentId} not found`); + } + return experiment; + } + + private validateVariants(variants: ExperimentVariant[]) { + if (!variants.length || variants.length < 2) { + throw new BadRequestException( + 'Experiments require at least two variants', + ); + } + + const seen = new Set(); + for (const variant of variants) { + if (!variant.key || seen.has(variant.key)) { + throw new BadRequestException( + 'Variant keys must be unique and non-empty', + ); + } + if (variant.weight <= 0) { + throw new BadRequestException('Variant weights must be positive'); + } + seen.add(variant.key); + } + } + + private pickVariant(experiment: SavingsExperiment, userId: string): string { + const digest = createHash('sha256') + .update(`${experiment.id}:${userId}`) + .digest('hex'); + const ratio = parseInt(digest.slice(0, 8), 16) / 0xffffffff; + const totalWeight = experiment.variants.reduce( + (sum, variant) => sum + variant.weight, + 0, + ); + + let cursor = 0; + for (const variant of experiment.variants) { + cursor += variant.weight / totalWeight; + if (ratio <= cursor) { + return variant.key; + } + } + + return experiment.variants[experiment.variants.length - 1].key; + } + + private buildVariantDashboard( + variant: ExperimentVariant, + assignments: SavingsExperimentAssignment[], + ): VariantDashboard { + const variantAssignments = assignments.filter( + (assignment) => assignment.variantKey === variant.key, + ); + const conversions = variantAssignments.filter( + (assignment) => assignment.convertedAt != null, + ); + const assignmentCount = variantAssignments.length; + const conversionCount = conversions.length; + + return { + key: variant.key, + assignments: assignmentCount, + conversions: conversionCount, + conversionRate: + assignmentCount > 0 + ? Number(((conversionCount / assignmentCount) * 100).toFixed(2)) + : 0, + totalConversionValue: Number( + conversions + .reduce( + (sum, assignment) => sum + Number(assignment.conversionValue ?? 0), + 0, + ) + .toFixed(2), + ), + }; + } + + private calculateSignificance( + baseline: VariantDashboard, + challenger: VariantDashboard, + confidenceLevel: number, + ) { + const p1 = baseline.assignments + ? baseline.conversions / baseline.assignments + : 0; + const p2 = challenger.assignments + ? challenger.conversions / challenger.assignments + : 0; + const pooled = + baseline.assignments + challenger.assignments > 0 + ? (baseline.conversions + challenger.conversions) / + (baseline.assignments + challenger.assignments) + : 0; + const standardError = Math.sqrt( + pooled * + (1 - pooled) * + ((baseline.assignments ? 1 / baseline.assignments : 0) + + (challenger.assignments ? 1 / challenger.assignments : 0)), + ); + const zScore = standardError > 0 ? (p2 - p1) / standardError : 0; + const confidence = Number( + (this.normalCdf(Math.abs(zScore)) * 100).toFixed(2), + ); + + return { + baselineVariant: baseline.key, + challengerVariant: challenger.key, + zScore: Number(zScore.toFixed(4)), + confidence, + statisticallySignificant: + confidence >= Number((confidenceLevel * 100).toFixed(2)) && + baseline.assignments >= 1 && + challenger.assignments >= 1, + }; + } + + private normalCdf(value: number): number { + return 0.5 * (1 + this.erf(value / Math.sqrt(2))); + } + + private erf(value: number): number { + const sign = value >= 0 ? 1 : -1; + const x = Math.abs(value); + const a1 = 0.254829592; + const a2 = -0.284496736; + const a3 = 1.421413741; + const a4 = -1.453152027; + const a5 = 1.061405429; + const p = 0.3275911; + const t = 1 / (1 + p * x); + const y = + 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x); + return sign * y; + } +} diff --git a/backend/src/modules/savings/savings.module.ts b/backend/src/modules/savings/savings.module.ts index 9cda42168..21d9d2b6e 100644 --- a/backend/src/modules/savings/savings.module.ts +++ b/backend/src/modules/savings/savings.module.ts @@ -13,6 +13,9 @@ import { WaitlistEntry } from './entities/waitlist-entry.entity'; import { WaitlistEvent } from './entities/waitlist-event.entity'; import { WaitlistService } from './waitlist.service'; import { WaitlistController } from './waitlist.controller'; +import { SavingsExperiment } from './entities/savings-experiment.entity'; +import { SavingsExperimentAssignment } from './entities/savings-experiment-assignment.entity'; +import { ExperimentsService } from './experiments.service'; @Module({ imports: [ @@ -25,10 +28,17 @@ import { WaitlistController } from './waitlist.controller'; User, WaitlistEntry, WaitlistEvent, + SavingsExperiment, + SavingsExperimentAssignment, ]), ], controllers: [SavingsController, WaitlistController], - providers: [SavingsService, PredictiveEvaluatorService, WaitlistService], - exports: [SavingsService, WaitlistService], + providers: [ + SavingsService, + PredictiveEvaluatorService, + WaitlistService, + ExperimentsService, + ], + exports: [SavingsService, WaitlistService, ExperimentsService], }) export class SavingsModule {}