diff --git a/README.md b/README.md index 49d6611..ffb70b5 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,14 @@ Click on any category to explore. - **Secure & Flexible:** Manages all sensitive keys, API credentials, and environment-specific settings through a `.env` file, keeping your secrets out of the codebase. > **Your Advantage:** Deploy your application across different environments (local, staging, production) safely and efficiently. +--- + +### 🔄 Automated Database Migrations +- **PR-Driven Schema Evolution:** Implements a robust, versioned database migration system that automatically applies schema changes to MongoDB on application startup. +- **Idempotent & Generic:** Each migration is idempotent and designed to handle schema evolution for *any* model in the database, ensuring data consistency across deployments. +- **Traceable Versioning:** Migrations are identified by their Pull Request merge date (`prDate` in `YYYYMMDDHHMMSS` format) for chronological execution, a concise `prSummary`, and a direct `prId` (GitHub PR ID) for full traceability. +> **Your Advantage:** Say goodbye to manual database updates! Your application gracefully handles schema changes, providing a professional and reliable mechanism for evolving your data models without breaking existing data, with clear links to the originating code changes. + ## 🔑 Licensing diff --git a/lib/src/config/app_dependencies.dart b/lib/src/config/app_dependencies.dart index 76dcff2..3de4fdc 100644 --- a/lib/src/config/app_dependencies.dart +++ b/lib/src/config/app_dependencies.dart @@ -6,11 +6,13 @@ import 'package:data_repository/data_repository.dart'; import 'package:email_repository/email_repository.dart'; import 'package:email_sendgrid/email_sendgrid.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/config/environment_config.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/database/migrations/all_migrations.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/rbac/permission_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/auth_token_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/country_query_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/dashboard_summary_service.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/services/database_migration_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/database_seeding_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/default_user_preference_limit_service.dart'; import 'package:flutter_news_app_api_server_full_source_code/src/services/jwt_auth_token_service.dart'; @@ -63,6 +65,7 @@ class AppDependencies { late final EmailRepository emailRepository; // Services + late final DatabaseMigrationService databaseMigrationService; late final TokenBlacklistService tokenBlacklistService; late final AuthTokenService authTokenService; late final VerificationCodeStorageService verificationCodeStorageService; @@ -92,7 +95,18 @@ class AppDependencies { await _mongoDbConnectionManager.init(EnvironmentConfig.databaseUrl); _log.info('MongoDB connection established.'); - // 2. Seed Database + // 2. Initialize and Run Database Migrations + databaseMigrationService = DatabaseMigrationService( + db: _mongoDbConnectionManager.db, + log: Logger('DatabaseMigrationService'), + migrations: + allMigrations, // From lib/src/database/migrations/all_migrations.dart + ); + await databaseMigrationService.init(); + _log.info('Database migrations applied.'); + + // 3. Seed Database + // This runs AFTER migrations to ensure the schema is up-to-date. final seedingService = DatabaseSeedingService( db: _mongoDbConnectionManager.db, log: Logger('DatabaseSeedingService'), @@ -100,7 +114,7 @@ class AppDependencies { await seedingService.seedInitialData(); _log.info('Database seeding complete.'); - // 3. Initialize Data Clients (MongoDB implementation) + // 4. Initialize Data Clients (MongoDB implementation) final headlineClient = DataMongodb( connectionManager: _mongoDbConnectionManager, modelName: 'headlines', diff --git a/lib/src/database/migration.dart b/lib/src/database/migration.dart new file mode 100644 index 0000000..1132bdf --- /dev/null +++ b/lib/src/database/migration.dart @@ -0,0 +1,61 @@ +import 'package:logging/logging.dart'; +import 'package:mongo_dart/mongo_dart.dart'; + +/// {@template migration} +/// An abstract base class for defining database migration scripts. +/// +/// Each concrete migration must extend this class and implement the [up] and +/// [down] methods. Migrations are identified by a unique [prDate] string +/// (following the `YYYYMMDDHHMMSS` format) and a [prSummary]. +/// +/// Implementations of [up] and [down] must be **idempotent**, meaning they +/// can be safely run multiple times without causing errors or incorrect data. +/// This is crucial for robust database schema evolution. +/// {@endtemplate} +abstract class Migration { + /// {@macro migration} + const Migration({ + required this.prDate, + required this.prSummary, + required this.prId, + }); + + /// The merge date and time of the Pull Request that introduced this + /// migration, in `YYYYMMDDHHMMSS` format (e.g., '20250924083500'). + /// + /// This serves as the unique, chronological identifier for the migration, + /// ensuring that migrations are applied in the correct order. + final String prDate; + + /// A concise summary of the changes introduced by the Pull Request that + /// this migration addresses. + /// + /// This provides a human-readable description of the migration's purpose. + final String prSummary; + + /// The unique identifier of the GitHub Pull Request that introduced the + /// schema changes addressed by this migration (e.g., '50'). + /// + /// This provides direct traceability, linking the database migration to the + /// specific code changes on GitHub. + final String prId; + + /// Applies the migration, performing necessary schema changes or data + /// transformations. + /// + /// This method is executed when the migration is run. It receives the + /// MongoDB [db] instance and a [Logger] for logging progress and errors. + /// + /// Implementations **must** be idempotent. + Future up(Db db, Logger log); + + /// Reverts the migration, undoing the changes made by the [up] method. + /// + /// This method is executed when a migration needs to be rolled back. It + /// receives the MongoDB [db] instance and a [Logger]. + /// + /// Implementations **must** be idempotent. While optional for simple + /// forward-only migrations, providing a `down` method is a best practice + /// for professional systems to enable rollback capabilities. + Future down(Db db, Logger log); +} diff --git a/lib/src/database/migrations/20250924084800__refactor_ad_config_to_role_based.dart b/lib/src/database/migrations/20250924084800__refactor_ad_config_to_role_based.dart new file mode 100644 index 0000000..6edf33e --- /dev/null +++ b/lib/src/database/migrations/20250924084800__refactor_ad_config_to_role_based.dart @@ -0,0 +1,160 @@ +// ignore_for_file: comment_references + +import 'package:core/core.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/database/migration.dart'; +import 'package:logging/logging.dart'; +import 'package:mongo_dart/mongo_dart.dart'; + +/// {@template refactor_ad_config_to_role_based} +/// A comprehensive migration to refactor the `adConfig` structure within +/// `RemoteConfig` documents to a new role-based `visibleTo` map approach. +/// +/// This migration addresses significant changes introduced by a PR (see +/// [gitHubPullRequest]) that aimed to enhance flexibility and maintainability +/// of ad configurations. It transforms old, role-specific ad frequency and +/// placement fields into new `visibleTo` maps for `FeedAdConfiguration`, +/// `ArticleAdConfiguration`, and `InterstitialAdConfiguration`. +/// +/// The migration ensures that existing `RemoteConfig` documents are updated +/// to conform to the latest model structure, preventing deserialization errors +/// and enabling granular control over ad display for different user roles. +/// {@endtemplate} +class RefactorAdConfigToRoleBased extends Migration { + /// {@macro refactor_ad_config_to_role_based} + RefactorAdConfigToRoleBased() + : super( + prDate: '20250924084800', + prSummary: 'Refactor adConfig to use role-based visibleTo maps', + prId: '50', + ); + + @override + Future up(Db db, Logger log) async { + log.info( + 'Applying migration PR#$prId (Date: $prDate): $prSummary.', + ); + + final remoteConfigCollection = db.collection('remote_configs'); + + // Define default FeedAdFrequencyConfig for roles + const defaultGuestFeedAdFrequency = FeedAdFrequencyConfig( + adFrequency: 5, + adPlacementInterval: 3, + ); + const defaultStandardUserFeedAdFrequency = FeedAdFrequencyConfig( + adFrequency: 10, + adPlacementInterval: 5, + ); + // Define default InterstitialAdFrequencyConfig for roles + const defaultGuestInterstitialAdFrequency = InterstitialAdFrequencyConfig( + transitionsBeforeShowingInterstitialAds: 5, + ); + const defaultStandardUserInterstitialAdFrequency = + InterstitialAdFrequencyConfig( + transitionsBeforeShowingInterstitialAds: 10, + ); + + // Define default ArticleAdSlot visibility for roles + final defaultArticleAdSlots = { + InArticleAdSlotType.aboveArticleContinueReadingButton.name: true, + InArticleAdSlotType.belowArticleContinueReadingButton.name: true, + }; + + final result = await remoteConfigCollection.updateMany( + // Find documents that still have the old structure (e.g., old frequency fields) + where.exists( + 'adConfig.feedAdConfiguration.frequencyConfig.guestAdFrequency', + ), + ModifierBuilder() + // --- FeedAdConfiguration Transformation --- + // Remove old frequencyConfig fields + ..unset('adConfig.feedAdConfiguration.frequencyConfig.guestAdFrequency') + ..unset( + 'adConfig.feedAdConfiguration.frequencyConfig.guestAdPlacementInterval', + ) + ..unset( + 'adConfig.feedAdConfiguration.frequencyConfig.authenticatedAdFrequency', + ) + ..unset( + 'adConfig.feedAdConfiguration.frequencyConfig.authenticatedAdPlacementInterval', + ) + ..unset( + 'adConfig.feedAdConfiguration.frequencyConfig.premiumAdFrequency', + ) + ..unset( + 'adConfig.feedAdConfiguration.frequencyConfig.premiumAdPlacementInterval', + ) + // Set the new visibleTo map for FeedAdConfiguration + ..set( + 'adConfig.feedAdConfiguration.visibleTo', + { + AppUserRole.guestUser.name: defaultGuestFeedAdFrequency.toJson(), + AppUserRole.standardUser.name: defaultStandardUserFeedAdFrequency + .toJson(), + }, + ) + // --- ArticleAdConfiguration Transformation --- + // Remove old inArticleAdSlotConfigurations list + ..unset('adConfig.articleAdConfiguration.inArticleAdSlotConfigurations') + // Set the new visibleTo map for ArticleAdConfiguration + ..set( + 'adConfig.articleAdConfiguration.visibleTo', + { + AppUserRole.guestUser.name: defaultArticleAdSlots, + AppUserRole.standardUser.name: defaultArticleAdSlots, + }, + ) + // --- InterstitialAdConfiguration Transformation --- + // Remove old feedInterstitialAdFrequencyConfig fields + ..unset( + 'adConfig.interstitialAdConfiguration.feedInterstitialAdFrequencyConfig.guestTransitionsBeforeShowingInterstitialAds', + ) + ..unset( + 'adConfig.interstitialAdConfiguration.feedInterstitialAdFrequencyConfig.standardUserTransitionsBeforeShowingInterstitialAds', + ) + ..unset( + 'adConfig.interstitialAdConfiguration.feedInterstitialAdFrequencyConfig.premiumUserTransitionsBeforeShowingInterstitialAds', + ) + // Set the new visibleTo map for InterstitialAdConfiguration + ..set( + 'adConfig.interstitialAdConfiguration.visibleTo', + { + AppUserRole.guestUser.name: defaultGuestInterstitialAdFrequency + .toJson(), + AppUserRole.standardUser.name: + defaultStandardUserInterstitialAdFrequency.toJson(), + }, + ), + ); + + log.info( + 'Updated ${result.nModified} remote_config documents ' + 'to new role-based adConfig structure.', + ); + } + + @override + Future down(Db db, Logger log) async { + log.warning( + 'Reverting migration: Revert adConfig to old structure ' + '(not recommended for production).', + ); + // This down migration is complex and primarily for development/testing rollback. + // Reverting to the old structure would require re-introducing the old fields + // and potentially losing data if the new structure was used. + // For simplicity in this example, we'll just unset the new fields. + final result = await db + .collection('remote_configs') + .updateMany( + where.exists('adConfig.feedAdConfiguration.visibleTo'), + ModifierBuilder() + ..unset('adConfig.feedAdConfiguration.visibleTo') + ..unset('adConfig.articleAdConfiguration.visibleTo') + ..unset('adConfig.interstitialAdConfiguration.visibleTo'), + ); + log.warning( + 'Reverted ${result.nModified} remote_config documents ' + 'by unsetting new adConfig fields.', + ); + } +} diff --git a/lib/src/database/migrations/all_migrations.dart b/lib/src/database/migrations/all_migrations.dart new file mode 100644 index 0000000..4c5a728 --- /dev/null +++ b/lib/src/database/migrations/all_migrations.dart @@ -0,0 +1,13 @@ +import 'package:flutter_news_app_api_server_full_source_code/src/database/migration.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/database/migrations/20250924084800__refactor_ad_config_to_role_based.dart'; +import 'package:flutter_news_app_api_server_full_source_code/src/services/database_migration_service.dart' + show DatabaseMigrationService; + +/// A central list of all database migrations to be applied. +/// +/// New migration classes should be added to this list. The +/// [DatabaseMigrationService] will automatically sort and apply them based on +/// their `prDate` property. +final List allMigrations = [ + RefactorAdConfigToRoleBased(), +]; diff --git a/lib/src/registry/model_registry.dart b/lib/src/registry/model_registry.dart index d4a3c7b..1e89284 100644 --- a/lib/src/registry/model_registry.dart +++ b/lib/src/registry/model_registry.dart @@ -459,4 +459,6 @@ typedef ModelRegistryMap = Map>; /// This makes the `modelRegistry` map available for injection into the /// request context via `context.read()`. It's primarily /// used by the middleware in `routes/api/v1/data/_middleware.dart`. -final Middleware modelRegistryProvider = provider((_) => modelRegistry); +final Middleware modelRegistryProvider = provider( + (_) => modelRegistry, +); diff --git a/lib/src/services/database_migration_service.dart b/lib/src/services/database_migration_service.dart new file mode 100644 index 0000000..aa4e826 --- /dev/null +++ b/lib/src/services/database_migration_service.dart @@ -0,0 +1,120 @@ +import 'package:flutter_news_app_api_server_full_source_code/src/database/migration.dart'; +import 'package:logging/logging.dart'; +import 'package:mongo_dart/mongo_dart.dart'; + +/// {@template database_migration_service} +/// A service responsible for managing and executing database migrations. +/// +/// This service discovers, sorts, and applies pending [Migration] scripts +/// to the MongoDB database. It tracks applied migrations in a dedicated +/// `migrations_history` collection to ensure idempotency and prevent +/// redundant execution. +/// +/// Migrations are identified by their PR merge date (YYYYMMDDHHMMSS) +/// and are always applied in chronological order. +/// {@endtemplate} +class DatabaseMigrationService { + /// {@macro database_migration_service} + DatabaseMigrationService({ + required Db db, + required Logger log, + required List migrations, + }) : _db = db, + _log = log, + _migrations = migrations; + + final Db _db; + final Logger _log; + final List _migrations; + + /// The name of the MongoDB collection used to track applied migrations. + /// This collection stores metadata about applied Pull Request migrations. + static const String _migrationsCollectionName = 'pr_migrations_history'; + + /// Initializes the migration service and applies any pending migrations. + /// + /// This method performs the following steps: + /// 1. Ensures the `pr_migrations_history` collection exists and has a unique + /// index on the `prDate` field. + /// 2. Fetches all previously applied migration PR dates from the database. + /// 3. Sorts the registered migrations by their `prDate` string. + /// 4. Iterates through the sorted migrations, applying only those that + /// have not yet been applied. + /// 5. Records each successfully applied migration's `prDate` and `prId` + /// in the `pr_migrations_history` collection. + Future init() async { + _log.info('Starting database migration process...'); + + await _ensureMigrationsCollectionAndIndex(); + + final appliedPrDates = await _getAppliedMigrationPrDates(); + _log.fine('Applied migration PR dates: $appliedPrDates'); + + // Sort migrations by prDate to ensure chronological application. + final sortedMigrations = [..._migrations] + ..sort((a, b) => a.prDate.compareTo(b.prDate)); + + for (final migration in sortedMigrations) { + if (!appliedPrDates.contains(migration.prDate)) { + _log.info( + 'Applying migration PR#${migration.prId} (Date: ${migration.prDate}): ' + '${migration.prSummary}', + ); + try { + await migration.up(_db, _log); + await _recordMigration(migration.prDate, migration.prId); + _log.info( + 'Successfully applied migration PR#${migration.prId} (Date: ${migration.prDate}).', + ); + } catch (e, s) { + _log.severe( + 'Failed to apply migration PR#${migration.prId} (Date: ${migration.prDate}): ' + '${migration.prSummary}', + e, + s, + ); + // Re-throw to halt application startup if a migration fails. + rethrow; + } + } else { + _log.fine( + 'Migration PR#${migration.prId} (Date: ${migration.prDate}) already applied. Skipping.', + ); + } + } + + _log.info('Database migration process completed.'); + } + + /// Ensures the `pr_migrations_history` collection exists and has a unique index + /// on the `prDate` field. + Future _ensureMigrationsCollectionAndIndex() async { + _log.fine('Ensuring pr_migrations_history collection and index...'); + final collection = _db.collection(_migrationsCollectionName); + await collection.createIndex( + key: 'prDate', + unique: true, + name: 'prDate_unique_index', + ); + _log.fine('Pr_migrations_history collection and index ensured.'); + } + + /// Retrieves a set of PR dates of all migrations that have already been + /// applied to the database. + Future> _getAppliedMigrationPrDates() async { + final collection = _db.collection(_migrationsCollectionName); + final documents = await collection.find().toList(); + return documents.map((doc) => doc['prDate'] as String).toSet(); + } + + /// Records a successfully applied migration in the `pr_migrations_history` + /// collection. + Future _recordMigration(String prDate, String prId) async { + final collection = _db.collection(_migrationsCollectionName); + await collection.insertOne({ + 'prDate': prDate, + 'prId': prId, + 'appliedAt': DateTime.now().toUtc(), + }); + } +} diff --git a/lib/src/services/database_seeding_service.dart b/lib/src/services/database_seeding_service.dart index 5121718..dd139da 100644 --- a/lib/src/services/database_seeding_service.dart +++ b/lib/src/services/database_seeding_service.dart @@ -128,7 +128,7 @@ class DatabaseSeedingService { final initialConfig = remoteConfigsFixturesData.first; // Ensure primaryAdPlatform is not 'demo' for initial setup - // sic its not intended for any use outside teh mobile client code. + // since its not intended for any use outside the mobile client. final productionReadyAdConfig = initialConfig.adConfig.copyWith( primaryAdPlatform: AdPlatformType.local, ); @@ -145,7 +145,10 @@ class DatabaseSeedingService { }); _log.info('Initial RemoteConfig created successfully.'); } else { - _log.info('RemoteConfig already exists. Skipping creation.'); + _log.info( + 'RemoteConfig already exists. Skipping creation. ' + 'Schema updates are handled by DatabaseMigrationService.', + ); } } on Exception catch (e, s) { _log.severe('Failed to seed RemoteConfig.', e, s);