From 3f14ed65df935f35e88c154fd7613c551596081f Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 27 Sep 2025 18:44:38 +0200 Subject: [PATCH 01/39] Added initial extensions for Database, Schema, Table, Column schema components They're yet dummy, but more will come then --- .../dumbo/src/core/schema/schemaComponent.ts | 16 ++++++++++++++++ .../database/pongoDatabaseSchemaComponent.ts | 7 ------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index bebf5768..8e858a48 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -35,3 +35,19 @@ export const schemaComponent = ( }, }; }; + +export type DatabaseSchemaComponent = + SchemaComponent; + +export type DatabaseSchemaSchemaComponent< + ComponentType extends string = string, +> = SchemaComponent; + +export type TableSchemaComponent = + SchemaComponent; + +export type ColumnSchemaComponent = + SchemaComponent; + +export type IndexSchemaComponent = + SchemaComponent; diff --git a/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts b/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts index 42aaab37..a01f0f48 100644 --- a/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts +++ b/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts @@ -30,13 +30,6 @@ export type PongoDatabaseSchemaComponent< ) => PongoCollectionSchemaComponent; }; -export type PongoDatabaseSchemaComponentFactory = < - DriverType extends DatabaseDriverType = DatabaseDriverType, ->( - driverType: DriverType, - existingCollections: PongoCollectionSchemaComponent[], -) => PongoDatabaseSchemaComponent; - export type PongoDatabaseSchemaComponentOptions< DriverType extends DatabaseDriverType = DatabaseDriverType, T extends Record = Record< From c8634a6096ff435c3a25ecd9edcd01ef2d8daaeb Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 27 Sep 2025 19:02:34 +0200 Subject: [PATCH 02/39] Added a first naive implementation of SchemaComponentMigrator We'll need to find a marter way for migration, probably hash calculated automatically as it's handled eventually or some id. --- .../dumbo/src/core/schema/schemaComponent.ts | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 8e858a48..14ce0c70 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -1,4 +1,9 @@ -import { type SQLMigration } from './migrations'; +import type { DatabaseDriverType, Dumbo } from '..'; +import { + runSQLMigrations, + type MigratorOptions, + type SQLMigration, +} from './migrations'; export type SchemaComponent = { schemaComponentType: ComponentType; @@ -51,3 +56,31 @@ export type ColumnSchemaComponent = export type IndexSchemaComponent = SchemaComponent; + +export type SchemaComponentMigrator = { + component: SchemaComponent; + run: (options?: Partial) => Promise; +}; + +export const SchemaComponentMigrator = ( + component: SchemaComponent, + dumbo: Dumbo, +): SchemaComponentMigrator => { + const completedMigrations: string[] = []; + + return { + component, + run: async (options) => { + //TODO: name is not the safest choice here, so we might want to add an id or hash instead + const pendingMigrations = component.migrations.filter( + (m) => !completedMigrations.includes(m.name), + ); + + if (pendingMigrations.length === 0) return; + + await runSQLMigrations(dumbo, pendingMigrations, options); + + completedMigrations.push(...pendingMigrations.map((m) => m.name)); + }, + }; +}; From 9718612eae77e3f32577a52cd1537ea3c4c3bd59 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Thu, 16 Oct 2025 12:01:47 +0200 Subject: [PATCH 03/39] Added drafts of schema components definitions --- .../dumbo/src/core/schema/schemaComponent.ts | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 14ce0c70..02ca73a6 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -41,21 +41,37 @@ export const schemaComponent = ( }; }; -export type DatabaseSchemaComponent = - SchemaComponent; +export type DatabaseSchemaComponent = + SchemaComponent<`sc:dumbo:database:${Kind}`> & { + databaseName: string; + schemas: ReadonlyArray; + }; export type DatabaseSchemaSchemaComponent< ComponentType extends string = string, -> = SchemaComponent; +> = SchemaComponent & { + schemaName: string; + tables: ReadonlyArray; +}; export type TableSchemaComponent = - SchemaComponent; + SchemaComponent & { + tableName: string; + columns: ReadonlyArray; + indexes: ReadonlyArray; + }; export type ColumnSchemaComponent = - SchemaComponent; + SchemaComponent & { + columnName: string; + }; export type IndexSchemaComponent = - SchemaComponent; + SchemaComponent & { + indexName: string; + columns: ReadonlyArray; + unique: boolean; + }; export type SchemaComponentMigrator = { component: SchemaComponent; From 1de4a0d3496b5de86a98c3fcc8463dab18f711db Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 16 Oct 2025 15:04:09 +0200 Subject: [PATCH 04/39] Added DatabaseSchemaComponent factory --- .../dumbo/src/core/schema/schemaComponent.ts | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 02ca73a6..d7f07b38 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -47,29 +47,52 @@ export type DatabaseSchemaComponent = schemas: ReadonlyArray; }; -export type DatabaseSchemaSchemaComponent< - ComponentType extends string = string, -> = SchemaComponent & { - schemaName: string; - tables: ReadonlyArray; +export const databaseSchemaComponent = ({ + kind, + databaseName, + schemas, + ...migrationsOrComponents +}: { + kind?: Kind; + databaseName: string; + schemas: ReadonlyArray; +} & SchemaComponentOptions): DatabaseSchemaComponent => { + const component = schemaComponent<`sc:dumbo:database:${Kind}`>( + `sc:dumbo:database:${(kind ?? 'regular') as Kind}`, + migrationsOrComponents, + ); + + return { + ...component, + databaseName, + get schemas() { + return schemas; + }, + }; }; -export type TableSchemaComponent = - SchemaComponent & { +export type DatabaseSchemaSchemaComponent = + SchemaComponent<`sc:dumbo:database_schema:${Kind}`> & { + schemaName: string; + tables: ReadonlyMap; + }; + +export type TableSchemaComponent = + SchemaComponent<`sc:dumbo:table:${Kind}`> & { tableName: string; - columns: ReadonlyArray; - indexes: ReadonlyArray; + columns: ReadonlyMap; + indexes: ReadonlyMap; }; -export type ColumnSchemaComponent = - SchemaComponent & { +export type ColumnSchemaComponent = + SchemaComponent<`sc:dumbo:column:${Kind}`> & { columnName: string; }; -export type IndexSchemaComponent = - SchemaComponent & { +export type IndexSchemaComponent = + SchemaComponent<`sc:dumbo:index:${Kind}`> & { indexName: string; - columns: ReadonlyArray; + columns: ReadonlyMap; unique: boolean; }; From 651a623f47723a201a606ac7c4423cb8c79cc13e Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 16 Oct 2025 15:25:13 +0200 Subject: [PATCH 05/39] Added additional options to SchemaComponent to make easier extending it --- .../dumbo/src/core/schema/schemaComponent.ts | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index d7f07b38..00239e0d 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -5,13 +5,18 @@ import { type SQLMigration, } from './migrations'; -export type SchemaComponent = { +export type SchemaComponent< + ComponentType extends string = string, + AdditionalOptions extends Record = Record, +> = { schemaComponentType: ComponentType; components: ReadonlyArray; migrations: ReadonlyArray; -}; +} & AdditionalOptions; -export type SchemaComponentOptions = +export type SchemaComponentOptions< + AdditionalOptions = Record, +> = ( | { migrations: ReadonlyArray; components?: never; @@ -23,7 +28,9 @@ export type SchemaComponentOptions = | { migrations?: never; components: ReadonlyArray; - }; + } +) & + Omit; export const schemaComponent = ( type: ComponentType, @@ -42,10 +49,13 @@ export const schemaComponent = ( }; export type DatabaseSchemaComponent = - SchemaComponent<`sc:dumbo:database:${Kind}`> & { - databaseName: string; - schemas: ReadonlyArray; - }; + SchemaComponent< + `sc:dumbo:database:${Kind}`, + { + databaseName: string; + schemas: ReadonlyArray; + } + >; export const databaseSchemaComponent = ({ kind, @@ -72,17 +82,23 @@ export const databaseSchemaComponent = ({ }; export type DatabaseSchemaSchemaComponent = - SchemaComponent<`sc:dumbo:database_schema:${Kind}`> & { - schemaName: string; - tables: ReadonlyMap; - }; + SchemaComponent< + `sc:dumbo:database_schema:${Kind}`, + { + schemaName: string; + tables: ReadonlyMap; + } + >; export type TableSchemaComponent = - SchemaComponent<`sc:dumbo:table:${Kind}`> & { - tableName: string; - columns: ReadonlyMap; - indexes: ReadonlyMap; - }; + SchemaComponent< + `sc:dumbo:table:${Kind}`, + { + tableName: string; + columns: ReadonlyMap; + indexes: ReadonlyMap; + } + >; export type ColumnSchemaComponent = SchemaComponent<`sc:dumbo:column:${Kind}`> & { From acb210cf350b3770d9b89d6ce9d8c56c4be36a65 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Tue, 21 Oct 2025 13:43:58 +0200 Subject: [PATCH 06/39] Added factories for SchemaComponents --- .../dumbo/src/core/schema/schemaComponent.ts | 161 +++++++++++++++--- 1 file changed, 136 insertions(+), 25 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 00239e0d..0276d592 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -7,24 +7,46 @@ import { export type SchemaComponent< ComponentType extends string = string, - AdditionalOptions extends Record = Record, + AdditionalData extends Record | undefined = undefined, > = { schemaComponentType: ComponentType; components: ReadonlyArray; migrations: ReadonlyArray; -} & AdditionalOptions; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + addComponent: (component: SchemaComponent) => void; + addMigration: (migration: SQLMigration) => void; +} & Omit< + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + AdditionalData extends undefined ? {} : AdditionalData, + | 'schemaComponentType' + | 'components' + | 'migrations' + | 'addComponent' + | 'addMigration' +>; + +export type ExtractAdditionalData = + T extends SchemaComponent ? Data : never; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnySchemaComponent = SchemaComponent; +export type AnySchemaComponentOfType = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + SchemaComponent; + +export type MigrationsAndComponents = { + migrations: ReadonlyArray; + components: ReadonlyArray; +}; export type SchemaComponentOptions< - AdditionalOptions = Record, + AdditionalOptions extends Record = Record, > = ( | { migrations: ReadonlyArray; components?: never; } - | { - migrations: ReadonlyArray; - components: ReadonlyArray; - } + | MigrationsAndComponents | { migrations?: never; components: ReadonlyArray; @@ -32,12 +54,31 @@ export type SchemaComponentOptions< ) & Omit; -export const schemaComponent = ( - type: ComponentType, - migrationsOrComponents: SchemaComponentOptions, -): SchemaComponent => { - const components = migrationsOrComponents.components ?? []; - const migrations = migrationsOrComponents.migrations ?? []; +export const schemaComponentFactory = < + SchemaComponentToCreate extends AnySchemaComponent = AnySchemaComponent, +>( + ...args: ExtractAdditionalData extends undefined + ? [ + type: SchemaComponentToCreate['schemaComponentType'], + migrationsOrComponents: SchemaComponentOptions, + ] + : [ + type: SchemaComponentToCreate['schemaComponentType'], + migrationsOrComponents: SchemaComponentOptions, + setup: (options: { + migrations: Array; + components: Array; + }) => ExtractAdditionalData, + ] +): SchemaComponentToCreate => { + const [type, migrationsOrComponents, setup] = args; + + const components: AnySchemaComponent[] = [ + ...(migrationsOrComponents.components ?? []), + ]; + const migrations: SQLMigration[] = [ + ...(migrationsOrComponents.migrations ?? []), + ]; return { schemaComponentType: type, @@ -45,12 +86,65 @@ export const schemaComponent = ( get migrations(): SQLMigration[] { return [...migrations, ...components.flatMap((c) => c.migrations)]; }, + addComponent: (component: SchemaComponent) => { + components.push(component); + migrations.push(...component.migrations); + }, + addMigration: (migration: SQLMigration) => { + migrations.push(migration); + }, + ...(setup + ? setup({ migrations, components }) + : ({} as ExtractAdditionalData)), + } satisfies SchemaComponent as unknown as SchemaComponentToCreate; +}; + +export type SchemaComponentType = `sc:${Kind}`; +export type DumboSchemaComponentType = + SchemaComponentType<`dumbo:${Kind}`>; + +export const schemaComponent = ( + type: ComponentType, + migrationsOrComponents: SchemaComponentOptions, +): SchemaComponent => + schemaComponentFactory(type, migrationsOrComponents); + +export const isSchemaComponentOfKind = < + SchemaComponentOfKind extends AnySchemaComponent = AnySchemaComponent, +>( + component: AnySchemaComponent, + kind: AnySchemaComponent['schemaComponentType'], +): component is SchemaComponentOfKind => + component.schemaComponentType.startsWith(kind); + +export const filterSchemaComponentsOfType = ( + components: ReadonlyArray, + typeGuard: (component: AnySchemaComponent) => component is T, +): T[] => components.filter(typeGuard); + +export const findSchemaComponentsOfType = ( + root: AnySchemaComponent, + typeGuard: (component: AnySchemaComponent) => component is T, +): T[] => { + const results: T[] = []; + + const traverse = (component: AnySchemaComponent) => { + if (typeGuard(component)) { + results.push(component); + } + for (const child of component.components) { + traverse(child); + } }; + + traverse(root); + + return results; }; export type DatabaseSchemaComponent = SchemaComponent< - `sc:dumbo:database:${Kind}`, + `sc:dumbo:database_schema:${Kind}`, { databaseName: string; schemas: ReadonlyArray; @@ -67,18 +161,35 @@ export const databaseSchemaComponent = ({ databaseName: string; schemas: ReadonlyArray; } & SchemaComponentOptions): DatabaseSchemaComponent => { - const component = schemaComponent<`sc:dumbo:database:${Kind}`>( - `sc:dumbo:database:${(kind ?? 'regular') as Kind}`, - migrationsOrComponents, - ); + kind ??= 'regular' as Kind; - return { - ...component, - databaseName, - get schemas() { - return schemas; + const { migrations, components } = migrationsOrComponents; + + return schemaComponentFactory( + `sc:dumbo:database_schema:${kind}`, + { + migrations: migrations ?? [], + components: [...(components ?? []), ...schemas], }, - }; + ({ + components, + }: { + migrations: ReadonlyArray; + components: ReadonlyArray; + }) => ({ + databaseName, + get schemas() { + return filterSchemaComponentsOfType< + DatabaseSchemaSchemaComponent + >(components, (c) => + isSchemaComponentOfKind>( + c, + `sc:dumbo:database_schema:${kind}`, + ), + ); + }, + }), + ); }; export type DatabaseSchemaSchemaComponent = @@ -119,7 +230,7 @@ export type SchemaComponentMigrator = { export const SchemaComponentMigrator = ( component: SchemaComponent, - dumbo: Dumbo, + dumbo: Dumbo, ): SchemaComponentMigrator => { const completedMigrations: string[] = []; From c846b5a68a0056d5511bb278dc59568c76fc2761 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sun, 26 Oct 2025 19:00:54 +0100 Subject: [PATCH 07/39] Removed overengineered kind in schema components and schema component factory We'll use composition instead --- .../dumbo/src/core/schema/schemaComponent.ts | 333 +++++++++++------- 1 file changed, 213 insertions(+), 120 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 0276d592..98673353 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -1,4 +1,4 @@ -import type { DatabaseDriverType, Dumbo } from '..'; +import { type DatabaseDriverType, type Dumbo } from '..'; import { runSQLMigrations, type MigratorOptions, @@ -34,45 +34,21 @@ export type AnySchemaComponentOfType = // eslint-disable-next-line @typescript-eslint/no-explicit-any SchemaComponent; -export type MigrationsAndComponents = { - migrations: ReadonlyArray; - components: ReadonlyArray; -}; - export type SchemaComponentOptions< AdditionalOptions extends Record = Record, -> = ( - | { - migrations: ReadonlyArray; - components?: never; - } - | MigrationsAndComponents - | { - migrations?: never; - components: ReadonlyArray; - } -) & - Omit; +> = { + migrations?: ReadonlyArray; + components?: ReadonlyArray; +} & Omit; -export const schemaComponentFactory = < - SchemaComponentToCreate extends AnySchemaComponent = AnySchemaComponent, ->( - ...args: ExtractAdditionalData extends undefined - ? [ - type: SchemaComponentToCreate['schemaComponentType'], - migrationsOrComponents: SchemaComponentOptions, - ] - : [ - type: SchemaComponentToCreate['schemaComponentType'], - migrationsOrComponents: SchemaComponentOptions, - setup: (options: { - migrations: Array; - components: Array; - }) => ExtractAdditionalData, - ] -): SchemaComponentToCreate => { - const [type, migrationsOrComponents, setup] = args; +export type SchemaComponentType = `sc:${Kind}`; +export type DumboSchemaComponentType = + SchemaComponentType<`dumbo:${Kind}`>; +export const schemaComponent = ( + type: ComponentType, + migrationsOrComponents: SchemaComponentOptions, +): SchemaComponent => { const components: AnySchemaComponent[] = [ ...(migrationsOrComponents.components ?? []), ]; @@ -93,29 +69,16 @@ export const schemaComponentFactory = < addMigration: (migration: SQLMigration) => { migrations.push(migration); }, - ...(setup - ? setup({ migrations, components }) - : ({} as ExtractAdditionalData)), - } satisfies SchemaComponent as unknown as SchemaComponentToCreate; + }; }; -export type SchemaComponentType = `sc:${Kind}`; -export type DumboSchemaComponentType = - SchemaComponentType<`dumbo:${Kind}`>; - -export const schemaComponent = ( - type: ComponentType, - migrationsOrComponents: SchemaComponentOptions, -): SchemaComponent => - schemaComponentFactory(type, migrationsOrComponents); - -export const isSchemaComponentOfKind = < - SchemaComponentOfKind extends AnySchemaComponent = AnySchemaComponent, +export const isSchemaComponentOfType = < + SchemaComponentOfType extends AnySchemaComponent = AnySchemaComponent, >( component: AnySchemaComponent, - kind: AnySchemaComponent['schemaComponentType'], -): component is SchemaComponentOfKind => - component.schemaComponentType.startsWith(kind); + type: AnySchemaComponent['schemaComponentType'], +): component is SchemaComponentOfType => + component.schemaComponentType.startsWith(type); export const filterSchemaComponentsOfType = ( components: ReadonlyArray, @@ -142,86 +105,216 @@ export const findSchemaComponentsOfType = ( return results; }; -export type DatabaseSchemaComponent = - SchemaComponent< - `sc:dumbo:database_schema:${Kind}`, - { - databaseName: string; - schemas: ReadonlyArray; - } - >; +export type DatabaseSchemaComponent = SchemaComponent< + `sc:dumbo:database`, + { + databaseName: string; + schemas: ReadonlyMap; + //addSchema: (schema: DatabaseSchemaSchemaComponent) => void; + } +>; + +export type DatabaseSchemaSchemaComponent = SchemaComponent< + `sc:dumbo:database_schema`, + { + schemaName: string; + tables: ReadonlyMap; + } +>; -export const databaseSchemaComponent = ({ - kind, +export type TableSchemaComponent = SchemaComponent< + `sc:dumbo:table`, + { + tableName: string; + columns: ReadonlyMap; + indexes: ReadonlyMap; + } +>; + +export type ColumnSchemaComponent = SchemaComponent<`sc:dumbo:column`> & { + columnName: string; +}; + +export type IndexSchemaComponent = SchemaComponent<`sc:dumbo:index`> & { + indexName: string; + columns: ReadonlyMap; + isUnique: boolean; +}; + +export const databaseSchemaComponent = ({ databaseName, - schemas, + schemaNames, ...migrationsOrComponents }: { - kind?: Kind; databaseName: string; - schemas: ReadonlyArray; -} & SchemaComponentOptions): DatabaseSchemaComponent => { - kind ??= 'regular' as Kind; + schemaNames?: string[]; +} & SchemaComponentOptions): DatabaseSchemaComponent => { + const migrations = migrationsOrComponents.migrations ?? []; + const components = migrationsOrComponents.components ?? []; + schemaNames = schemaNames ?? []; - const { migrations, components } = migrationsOrComponents; + const sc = schemaComponent(`sc:dumbo:database`, { + migrations, + components: [ + ...components, + ...schemaNames.map((schemaName) => + databaseSchemaSchemaComponent({ schemaName }), + ), + ], + }); - return schemaComponentFactory( - `sc:dumbo:database_schema:${kind}`, - { - migrations: migrations ?? [], - components: [...(components ?? []), ...schemas], - }, - ({ - components, - }: { - migrations: ReadonlyArray; - components: ReadonlyArray; - }) => ({ - databaseName, - get schemas() { - return filterSchemaComponentsOfType< - DatabaseSchemaSchemaComponent - >(components, (c) => - isSchemaComponentOfKind>( + return { + ...sc, + databaseName, + get schemas() { + return filterSchemaComponentsOfType( + sc.components, + (c) => + isSchemaComponentOfType( c, - `sc:dumbo:database_schema:${kind}`, + `sc:dumbo:database`, ), - ); - }, - }), - ); + ).reduce((map, schema) => { + map.set(schema.schemaName, schema); + return map; + }, new Map()); + }, + }; }; -export type DatabaseSchemaSchemaComponent = - SchemaComponent< - `sc:dumbo:database_schema:${Kind}`, - { - schemaName: string; - tables: ReadonlyMap; - } - >; - -export type TableSchemaComponent = - SchemaComponent< - `sc:dumbo:table:${Kind}`, - { - tableName: string; - columns: ReadonlyMap; - indexes: ReadonlyMap; - } - >; +export const databaseSchemaSchemaComponent = ({ + schemaName, + tableNames, + ...migrationsOrComponents +}: { + schemaName: string; + tableNames?: string[]; +} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { + const migrations = migrationsOrComponents.migrations ?? []; + const components = migrationsOrComponents.components ?? []; + tableNames = tableNames ?? []; + + const sc = schemaComponent(`sc:dumbo:database_schema`, { + migrations, + components: [ + ...components, + ...tableNames.map((tableName) => tableSchemaComponent({ tableName })), + ], + }); -export type ColumnSchemaComponent = - SchemaComponent<`sc:dumbo:column:${Kind}`> & { - columnName: string; + return { + ...sc, + schemaName, + get tables() { + return filterSchemaComponentsOfType( + sc.components, + (c) => + isSchemaComponentOfType(c, `sc:dumbo:table`), + ).reduce((map, table) => { + map.set(table.tableName, table); + return map; + }, new Map()); + }, }; +}; -export type IndexSchemaComponent = - SchemaComponent<`sc:dumbo:index:${Kind}`> & { - indexName: string; - columns: ReadonlyMap; - unique: boolean; +export const tableSchemaComponent = ({ + tableName, + columnNames, + ...migrationsOrComponents +}: { + tableName: string; + columnNames?: string[]; +} & SchemaComponentOptions): TableSchemaComponent => { + const migrations = migrationsOrComponents.migrations ?? []; + const components = migrationsOrComponents.components ?? []; + columnNames = columnNames ?? []; + + const sc = schemaComponent(`sc:dumbo:table`, { + migrations, + components: [ + ...components, + ...columnNames.map((columnName) => columnSchemaComponent({ columnName })), + ], + }); + + return { + ...sc, + tableName, + get columns() { + return filterSchemaComponentsOfType( + sc.components, + (c) => + isSchemaComponentOfType(c, `sc:dumbo:column`), + ).reduce((map, column) => { + map.set(column.columnName, column); + return map; + }, new Map()); + }, + get indexes() { + return filterSchemaComponentsOfType( + sc.components, + (c) => + isSchemaComponentOfType(c, `sc:dumbo:index`), + ).reduce((map, index) => { + map.set(index.indexName, index); + return map; + }, new Map()); + }, }; +}; + +export const columnSchemaComponent = ({ + columnName, + ...migrationsOrComponents +}: { + columnName: string; +} & SchemaComponentOptions): ColumnSchemaComponent => { + const sc = schemaComponent(`sc:dumbo:column`, migrationsOrComponents); + + return { + ...sc, + columnName, + }; +}; + +export const indexSchemaComponent = ({ + indexName, + columnNames, + isUnique, + ...migrationsOrComponents +}: { + indexName: string; + columnNames: string[]; + isUnique: boolean; +} & SchemaComponentOptions): IndexSchemaComponent => { + const migrations = migrationsOrComponents.migrations ?? []; + const components = migrationsOrComponents.components ?? []; + + const sc = schemaComponent(`sc:dumbo:index`, { + migrations, + components: [ + ...components, + ...columnNames.map((columnName) => columnSchemaComponent({ columnName })), + ], + }); + + return { + ...sc, + indexName, + get columns() { + return filterSchemaComponentsOfType( + sc.components, + (c) => + isSchemaComponentOfType(c, `sc:dumbo:column`), + ).reduce((map, column) => { + map.set(column.columnName, column); + return map; + }, new Map()); + }, + isUnique, + }; +}; export type SchemaComponentMigrator = { component: SchemaComponent; From b90e4228fa6035cf46f9d07f97dbf90dff2a0ef1 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 11:25:37 +0100 Subject: [PATCH 08/39] Refactored Schema Component to use urn key instead of just component type --- .../dumbo/src/core/schema/schemaComponent.ts | 259 ++++++++++-------- 1 file changed, 139 insertions(+), 120 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 98673353..5f5b8704 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -5,12 +5,51 @@ import { type SQLMigration, } from './migrations'; +export type DatabaseURNType = 'sc:dumbo:database'; +export type SchemaURNType = 'sc:dumbo:schema'; +export type TableURNType = 'sc:dumbo:table'; +export type ColumnURNType = 'sc:dumbo:column'; +export type IndexURNType = 'sc:dumbo:index'; + +export type DatabaseURN = `${DatabaseURNType}:${string}`; +export type SchemaURN = `${SchemaURNType}:${string}`; +export type TableURN = `${TableURNType}:${string}`; +export type ColumnURN = `${ColumnURNType}:${string}`; +export type IndexURN = `${IndexURNType}:${string}`; + +export const schemaComponentURN = { + database: { + type: 'sc:dumbo:database' as DatabaseURNType, + build: (name: string): DatabaseURN => `sc:dumbo:database:${name}`, + }, + schema: { + type: 'sc:dumbo:schema' as SchemaURNType, + build: (name: string): SchemaURN => `sc:dumbo:schema:${name}`, + }, + table: { + type: 'sc:dumbo:table' as TableURNType, + build: (name: string): TableURN => `sc:dumbo:table:${name}`, + }, + column: { + type: 'sc:dumbo:column' as ColumnURNType, + build: (name: string): ColumnURN => `sc:dumbo:column:${name}`, + }, + index: { + type: 'sc:dumbo:index' as IndexURNType, + build: (name: string): IndexURN => `sc:dumbo:index:${name}`, + }, + extractName: (urn: string): string => { + const parts = urn.split(':'); + return parts[parts.length - 1] || ''; + }, +} as const; + export type SchemaComponent< - ComponentType extends string = string, + ComponentKey extends string = string, AdditionalData extends Record | undefined = undefined, > = { - schemaComponentType: ComponentType; - components: ReadonlyArray; + schemaComponentKey: ComponentKey; + components: ReadonlyMap; migrations: ReadonlyArray; // eslint-disable-next-line @typescript-eslint/no-explicit-any addComponent: (component: SchemaComponent) => void; @@ -18,7 +57,7 @@ export type SchemaComponent< } & Omit< // eslint-disable-next-line @typescript-eslint/no-empty-object-type AdditionalData extends undefined ? {} : AdditionalData, - | 'schemaComponentType' + | 'schemaComponentKey' | 'components' | 'migrations' | 'addComponent' @@ -45,25 +84,27 @@ export type SchemaComponentType = `sc:${Kind}`; export type DumboSchemaComponentType = SchemaComponentType<`dumbo:${Kind}`>; -export const schemaComponent = ( - type: ComponentType, - migrationsOrComponents: SchemaComponentOptions, -): SchemaComponent => { - const components: AnySchemaComponent[] = [ - ...(migrationsOrComponents.components ?? []), - ]; - const migrations: SQLMigration[] = [ - ...(migrationsOrComponents.migrations ?? []), - ]; +export const schemaComponent = ( + key: ComponentKey, + options: SchemaComponentOptions, +): SchemaComponent => { + const componentsMap = new Map( + options.components?.map((comp) => [comp.schemaComponentKey, comp]), + ); + + const migrations: SQLMigration[] = [...(options.migrations ?? [])]; return { - schemaComponentType: type, - components, + schemaComponentKey: key, + components: componentsMap, get migrations(): SQLMigration[] { - return [...migrations, ...components.flatMap((c) => c.migrations)]; + return [ + ...migrations, + ...Array.from(componentsMap.values()).flatMap((c) => c.migrations), + ]; }, addComponent: (component: SchemaComponent) => { - components.push(component); + componentsMap.set(component.schemaComponentKey, component); migrations.push(...component.migrations); }, addMigration: (migration: SQLMigration) => { @@ -76,26 +117,32 @@ export const isSchemaComponentOfType = < SchemaComponentOfType extends AnySchemaComponent = AnySchemaComponent, >( component: AnySchemaComponent, - type: AnySchemaComponent['schemaComponentType'], + prefix: string, ): component is SchemaComponentOfType => - component.schemaComponentType.startsWith(type); + component.schemaComponentKey.startsWith(prefix); export const filterSchemaComponentsOfType = ( - components: ReadonlyArray, - typeGuard: (component: AnySchemaComponent) => component is T, -): T[] => components.filter(typeGuard); + components: ReadonlyMap, + prefix: string, +): ReadonlyMap => { + return new Map( + Array.from(components.entries()) + .filter(([urn]) => urn.startsWith(prefix)) + .map(([urn, component]) => [urn, component as T]), + ); +}; export const findSchemaComponentsOfType = ( root: AnySchemaComponent, - typeGuard: (component: AnySchemaComponent) => component is T, + prefix: string, ): T[] => { const results: T[] = []; const traverse = (component: AnySchemaComponent) => { - if (typeGuard(component)) { - results.push(component); + if (component.schemaComponentKey.startsWith(prefix)) { + results.push(component as T); } - for (const child of component.components) { + for (const child of component.components.values()) { traverse(child); } }; @@ -106,40 +153,45 @@ export const findSchemaComponentsOfType = ( }; export type DatabaseSchemaComponent = SchemaComponent< - `sc:dumbo:database`, - { + DatabaseURN, + Readonly<{ databaseName: string; schemas: ReadonlyMap; - //addSchema: (schema: DatabaseSchemaSchemaComponent) => void; - } + }> >; export type DatabaseSchemaSchemaComponent = SchemaComponent< - `sc:dumbo:database_schema`, - { + SchemaURN, + Readonly<{ schemaName: string; tables: ReadonlyMap; - } + }> >; export type TableSchemaComponent = SchemaComponent< - `sc:dumbo:table`, - { + TableURN, + Readonly<{ tableName: string; columns: ReadonlyMap; indexes: ReadonlyMap; - } + }> >; -export type ColumnSchemaComponent = SchemaComponent<`sc:dumbo:column`> & { - columnName: string; -}; +export type ColumnSchemaComponent = SchemaComponent< + ColumnURN, + Readonly<{ + columnName: string; + }> +>; -export type IndexSchemaComponent = SchemaComponent<`sc:dumbo:index`> & { - indexName: string; - columns: ReadonlyMap; - isUnique: boolean; -}; +export type IndexSchemaComponent = SchemaComponent< + IndexURN, + Readonly<{ + indexName: string; + columns: ReadonlyMap; + isUnique: boolean; + }> +>; export const databaseSchemaComponent = ({ databaseName, @@ -149,18 +201,14 @@ export const databaseSchemaComponent = ({ databaseName: string; schemaNames?: string[]; } & SchemaComponentOptions): DatabaseSchemaComponent => { - const migrations = migrationsOrComponents.migrations ?? []; - const components = migrationsOrComponents.components ?? []; - schemaNames = schemaNames ?? []; - - const sc = schemaComponent(`sc:dumbo:database`, { - migrations, - components: [ - ...components, - ...schemaNames.map((schemaName) => - databaseSchemaSchemaComponent({ schemaName }), - ), - ], + const schemas = + schemaNames?.map((schemaName) => + databaseSchemaSchemaComponent({ schemaName }), + ) ?? []; + + const sc = schemaComponent(schemaComponentURN.database.build(databaseName), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...schemas], }); return { @@ -169,15 +217,8 @@ export const databaseSchemaComponent = ({ get schemas() { return filterSchemaComponentsOfType( sc.components, - (c) => - isSchemaComponentOfType( - c, - `sc:dumbo:database`, - ), - ).reduce((map, schema) => { - map.set(schema.schemaName, schema); - return map; - }, new Map()); + schemaComponentURN.schema.type, + ); }, }; }; @@ -190,16 +231,12 @@ export const databaseSchemaSchemaComponent = ({ schemaName: string; tableNames?: string[]; } & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { - const migrations = migrationsOrComponents.migrations ?? []; - const components = migrationsOrComponents.components ?? []; - tableNames = tableNames ?? []; - - const sc = schemaComponent(`sc:dumbo:database_schema`, { - migrations, - components: [ - ...components, - ...tableNames.map((tableName) => tableSchemaComponent({ tableName })), - ], + const tables = + tableNames?.map((tableName) => tableSchemaComponent({ tableName })) ?? []; + + const sc = schemaComponent(schemaComponentURN.schema.build(schemaName), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...tables], }); return { @@ -208,12 +245,8 @@ export const databaseSchemaSchemaComponent = ({ get tables() { return filterSchemaComponentsOfType( sc.components, - (c) => - isSchemaComponentOfType(c, `sc:dumbo:table`), - ).reduce((map, table) => { - map.set(table.tableName, table); - return map; - }, new Map()); + schemaComponentURN.table.type, + ); }, }; }; @@ -226,16 +259,13 @@ export const tableSchemaComponent = ({ tableName: string; columnNames?: string[]; } & SchemaComponentOptions): TableSchemaComponent => { - const migrations = migrationsOrComponents.migrations ?? []; - const components = migrationsOrComponents.components ?? []; - columnNames = columnNames ?? []; - - const sc = schemaComponent(`sc:dumbo:table`, { - migrations, - components: [ - ...components, - ...columnNames.map((columnName) => columnSchemaComponent({ columnName })), - ], + const columns = + columnNames?.map((columnName) => columnSchemaComponent({ columnName })) ?? + []; + + const sc = schemaComponent(schemaComponentURN.table.build(tableName), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...columns], }); return { @@ -244,22 +274,14 @@ export const tableSchemaComponent = ({ get columns() { return filterSchemaComponentsOfType( sc.components, - (c) => - isSchemaComponentOfType(c, `sc:dumbo:column`), - ).reduce((map, column) => { - map.set(column.columnName, column); - return map; - }, new Map()); + schemaComponentURN.column.type, + ); }, get indexes() { return filterSchemaComponentsOfType( sc.components, - (c) => - isSchemaComponentOfType(c, `sc:dumbo:index`), - ).reduce((map, index) => { - map.set(index.indexName, index); - return map; - }, new Map()); + schemaComponentURN.index.type, + ); }, }; }; @@ -270,7 +292,10 @@ export const columnSchemaComponent = ({ }: { columnName: string; } & SchemaComponentOptions): ColumnSchemaComponent => { - const sc = schemaComponent(`sc:dumbo:column`, migrationsOrComponents); + const sc = schemaComponent( + schemaComponentURN.column.build(columnName), + migrationsOrComponents, + ); return { ...sc, @@ -288,15 +313,13 @@ export const indexSchemaComponent = ({ columnNames: string[]; isUnique: boolean; } & SchemaComponentOptions): IndexSchemaComponent => { - const migrations = migrationsOrComponents.migrations ?? []; - const components = migrationsOrComponents.components ?? []; - - const sc = schemaComponent(`sc:dumbo:index`, { - migrations, - components: [ - ...components, - ...columnNames.map((columnName) => columnSchemaComponent({ columnName })), - ], + const columns = columnNames.map((columnName) => + columnSchemaComponent({ columnName }), + ); + + const sc = schemaComponent(schemaComponentURN.index.build(indexName), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...columns], }); return { @@ -305,12 +328,8 @@ export const indexSchemaComponent = ({ get columns() { return filterSchemaComponentsOfType( sc.components, - (c) => - isSchemaComponentOfType(c, `sc:dumbo:column`), - ).reduce((map, column) => { - map.set(column.columnName, column); - return map; - }, new Map()); + schemaComponentURN.column.type, + ); }, isUnique, }; From 06e7d045ffe36b41669a3e82c2f935e2cfbae486 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 11:36:00 +0100 Subject: [PATCH 09/39] Added mapping for the specific schema component nested components to use their names isntead of urns Thanks to that we'll be able to easier define the schema in the pongo like way. Made also migrations unique by adding schema component key to them. --- .../dumbo/src/core/schema/schemaComponent.ts | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 5f5b8704..4d4d1112 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -124,13 +124,21 @@ export const isSchemaComponentOfType = < export const filterSchemaComponentsOfType = ( components: ReadonlyMap, prefix: string, -): ReadonlyMap => { - return new Map( +): ReadonlyMap => mapSchemaComponentsOfType(components, prefix); + +export const mapSchemaComponentsOfType = ( + components: ReadonlyMap, + prefix: string, + keyMapper?: (component: T) => string, +): ReadonlyMap => + new Map( Array.from(components.entries()) .filter(([urn]) => urn.startsWith(prefix)) - .map(([urn, component]) => [urn, component as T]), + .map(([urn, component]) => [ + keyMapper ? keyMapper(component) : urn, + component as T, + ]), ); -}; export const findSchemaComponentsOfType = ( root: AnySchemaComponent, @@ -215,9 +223,10 @@ export const databaseSchemaComponent = ({ ...sc, databaseName, get schemas() { - return filterSchemaComponentsOfType( + return mapSchemaComponentsOfType( sc.components, schemaComponentURN.schema.type, + (c) => c.schemaName, ); }, }; @@ -243,9 +252,10 @@ export const databaseSchemaSchemaComponent = ({ ...sc, schemaName, get tables() { - return filterSchemaComponentsOfType( + return mapSchemaComponentsOfType( sc.components, schemaComponentURN.table.type, + (c) => c.tableName, ); }, }; @@ -272,15 +282,17 @@ export const tableSchemaComponent = ({ ...sc, tableName, get columns() { - return filterSchemaComponentsOfType( + return mapSchemaComponentsOfType( sc.components, schemaComponentURN.column.type, + (c) => c.columnName, ); }, get indexes() { - return filterSchemaComponentsOfType( + return mapSchemaComponentsOfType( sc.components, schemaComponentURN.index.type, + (c) => c.indexName, ); }, }; @@ -326,9 +338,10 @@ export const indexSchemaComponent = ({ ...sc, indexName, get columns() { - return filterSchemaComponentsOfType( + return mapSchemaComponentsOfType( sc.components, schemaComponentURN.column.type, + (c) => c.columnName, ); }, isUnique, @@ -349,16 +362,22 @@ export const SchemaComponentMigrator = ( return { component, run: async (options) => { - //TODO: name is not the safest choice here, so we might want to add an id or hash instead const pendingMigrations = component.migrations.filter( - (m) => !completedMigrations.includes(m.name), + (m) => + !completedMigrations.includes( + `${component.schemaComponentKey}:${m.name}`, + ), ); if (pendingMigrations.length === 0) return; await runSQLMigrations(dumbo, pendingMigrations, options); - completedMigrations.push(...pendingMigrations.map((m) => m.name)); + completedMigrations.push( + ...pendingMigrations.map( + (m) => `${component.schemaComponentKey}:${m.name}`, + ), + ); }, }; }; From 64660af0483fb9838567c29a118cb630b9a1422c Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 12:49:05 +0100 Subject: [PATCH 10/39] Moved schema components to dedicated files --- .../components/columnSchemaComponent.ts | 35 +++ .../components/databaseSchemaComponent.ts | 56 ++++ .../databaseSchemaSchemaComponent.ts | 57 +++++ .../dumbo/src/core/schema/components/index.ts | 23 ++ .../schema/components/indexSchemaComponent.ts | 59 +++++ .../schema/components/tableSchemaComponent.ts | 67 +++++ src/packages/dumbo/src/core/schema/index.ts | 1 + .../dumbo/src/core/schema/migrations.ts | 21 +- .../dumbo/src/core/schema/schemaComponent.ts | 242 ++---------------- 9 files changed, 317 insertions(+), 244 deletions(-) create mode 100644 src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts create mode 100644 src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts create mode 100644 src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts create mode 100644 src/packages/dumbo/src/core/schema/components/index.ts create mode 100644 src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts create mode 100644 src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts new file mode 100644 index 00000000..c96e6592 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -0,0 +1,35 @@ +import { + schemaComponent, + type SchemaComponent, + type SchemaComponentOptions, +} from '../schemaComponent'; + +export type ColumnURNType = 'sc:dumbo:column'; +export type ColumnURN = `${ColumnURNType}:${string}`; + +export const ColumnURNType: ColumnURNType = 'sc:dumbo:column'; +export const ColumnURN = ({ name }: { name: string }): ColumnURN => + `${ColumnURNType}:${name}`; + +export type ColumnSchemaComponent = SchemaComponent< + ColumnURN, + Readonly<{ + columnName: string; + }> +>; +export const columnSchemaComponent = ({ + columnName, + ...migrationsOrComponents +}: { + columnName: string; +} & SchemaComponentOptions): ColumnSchemaComponent => { + const sc = schemaComponent( + ColumnURN({ name: columnName }), + migrationsOrComponents, + ); + + return { + ...sc, + columnName, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts new file mode 100644 index 00000000..0b032bb7 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts @@ -0,0 +1,56 @@ +import { + mapSchemaComponentsOfType, + schemaComponent, + type SchemaComponent, + type SchemaComponentOptions, +} from '../schemaComponent'; +import { + databaseSchemaSchemaComponent, + type DatabaseSchemaSchemaComponent, +} from './databaseSchemaSchemaComponent'; + +export type DatabaseURNType = 'sc:dumbo:database'; +export type DatabaseURN = `${DatabaseURNType}:${string}`; + +export const DatabaseURNType: DatabaseURNType = 'sc:dumbo:database'; +export const DatabaseURN = ({ name }: { name: string }): DatabaseURN => + `${DatabaseURNType}:${name}`; + +export type DatabaseSchemaComponent = SchemaComponent< + DatabaseURN, + Readonly<{ + databaseName: string; + schemas: ReadonlyMap; + }> +>; + +export const databaseSchemaComponent = ({ + databaseName, + schemaNames, + ...migrationsOrComponents +}: { + databaseName: string; + schemaNames?: string[]; +} & SchemaComponentOptions): DatabaseSchemaComponent => { + const schemas = + schemaNames?.map((schemaName) => + databaseSchemaSchemaComponent({ schemaName }), + ) ?? []; + + const sc = schemaComponent(DatabaseURN({ name: databaseName }), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...schemas], + }); + + return { + ...sc, + databaseName, + get schemas() { + return mapSchemaComponentsOfType( + sc.components, + DatabaseURNType, + (c) => c.schemaName, + ); + }, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts new file mode 100644 index 00000000..15c51d4c --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -0,0 +1,57 @@ +import { + mapSchemaComponentsOfType, + schemaComponent, + type SchemaComponent, + type SchemaComponentOptions, +} from '../schemaComponent'; +import { + tableSchemaComponent, + type TableSchemaComponent, +} from './tableSchemaComponent'; + +export type DatabaseSchemaURNType = 'sc:dumbo:schema'; +export type DatabaseSchemaURN = `${DatabaseSchemaURNType}:${string}`; + +export const DatabaseSchemaURNType: DatabaseSchemaURNType = 'sc:dumbo:schema'; +export const DatabaseSchemaURN = ({ + name, +}: { + name: string; +}): DatabaseSchemaURN => `${DatabaseSchemaURNType}:${name}`; + +export type DatabaseSchemaSchemaComponent = SchemaComponent< + DatabaseSchemaURN, + Readonly<{ + schemaName: string; + tables: ReadonlyMap; + }> +>; + +export const databaseSchemaSchemaComponent = ({ + schemaName, + tableNames, + ...migrationsOrComponents +}: { + schemaName: string; + tableNames?: string[]; +} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { + const tables = + tableNames?.map((tableName) => tableSchemaComponent({ tableName })) ?? []; + + const sc = schemaComponent(DatabaseSchemaURN({ name: schemaName }), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...tables], + }); + + return { + ...sc, + schemaName, + get tables() { + return mapSchemaComponentsOfType( + sc.components, + DatabaseSchemaURNType, + (c) => c.tableName, + ); + }, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts new file mode 100644 index 00000000..fe39d7a8 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -0,0 +1,23 @@ +import { ColumnURN } from './columnSchemaComponent'; +import { DatabaseURN } from './databaseSchemaComponent'; +import { DatabaseSchemaURN } from './databaseSchemaSchemaComponent'; +import { IndexURN } from './indexSchemaComponent'; +import { TableURN } from './tableSchemaComponent'; + +export * from './columnSchemaComponent'; +export * from './databaseSchemaComponent'; +export * from './databaseSchemaSchemaComponent'; +export * from './indexSchemaComponent'; +export * from './tableSchemaComponent'; + +export const schemaComponentURN = { + database: DatabaseURN, + schema: DatabaseSchemaURN, + table: TableURN, + column: ColumnURN, + index: IndexURN, + extractName: (urn: string): string => { + const parts = urn.split(':'); + return parts[parts.length - 1] || ''; + }, +} as const; diff --git a/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts new file mode 100644 index 00000000..dbedd9a1 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts @@ -0,0 +1,59 @@ +import { + mapSchemaComponentsOfType, + schemaComponent, + type SchemaComponent, + type SchemaComponentOptions, +} from '../schemaComponent'; +import { + columnSchemaComponent, + type ColumnSchemaComponent, +} from './columnSchemaComponent'; + +export type IndexURNType = 'sc:dumbo:index'; +export type IndexURN = `${IndexURNType}:${string}`; + +export type IndexSchemaComponent = SchemaComponent< + IndexURN, + Readonly<{ + indexName: string; + columns: ReadonlyMap; + isUnique: boolean; + }> +>; + +export const IndexURNType: IndexURNType = 'sc:dumbo:index'; +export const IndexURN = ({ name }: { name: string }): IndexURN => + `${IndexURNType}:${name}`; + +export const indexSchemaComponent = ({ + indexName, + columnNames, + isUnique, + ...migrationsOrComponents +}: { + indexName: string; + columnNames: string[]; + isUnique: boolean; +} & SchemaComponentOptions): IndexSchemaComponent => { + const columns = columnNames.map((columnName) => + columnSchemaComponent({ columnName }), + ); + + const sc = schemaComponent(IndexURN({ name: indexName }), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...columns], + }); + + return { + ...sc, + indexName, + get columns() { + return mapSchemaComponentsOfType( + sc.components, + IndexURNType, + (c) => c.columnName, + ); + }, + isUnique, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts new file mode 100644 index 00000000..9a060159 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -0,0 +1,67 @@ +import { + mapSchemaComponentsOfType, + schemaComponent, + type SchemaComponent, + type SchemaComponentOptions, +} from '../schemaComponent'; +import { + columnSchemaComponent, + type ColumnSchemaComponent, +} from './columnSchemaComponent'; +import { + IndexURNType, + type IndexSchemaComponent, +} from './indexSchemaComponent'; + +export type TableURNType = 'sc:dumbo:table'; +export type TableURN = `${TableURNType}:${string}`; + +export const TableURNType: TableURNType = 'sc:dumbo:table'; +export const TableURN = ({ name }: { name: string }): TableURN => + `${TableURNType}:${name}`; + +export type TableSchemaComponent = SchemaComponent< + TableURN, + Readonly<{ + tableName: string; + columns: ReadonlyMap; + indexes: ReadonlyMap; + }> +>; + +export const tableSchemaComponent = ({ + tableName, + columnNames, + ...migrationsOrComponents +}: { + tableName: string; + columnNames?: string[]; +} & SchemaComponentOptions): TableSchemaComponent => { + const columns = + columnNames?.map((columnName) => columnSchemaComponent({ columnName })) ?? + []; + + const sc = schemaComponent(TableURN({ name: tableName }), { + migrations: migrationsOrComponents.migrations ?? [], + components: [...(migrationsOrComponents.components ?? []), ...columns], + }); + + return { + ...sc, + tableName, + get columns() { + return mapSchemaComponentsOfType( + sc.components, + TableURNType, + (c) => c.columnName, + ); + }, + get indexes() { + return mapSchemaComponentsOfType( + sc.components, + IndexURNType, + (c) => c.indexName, + ); + }, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/index.ts b/src/packages/dumbo/src/core/schema/index.ts index 582a76eb..17b3d38d 100644 --- a/src/packages/dumbo/src/core/schema/index.ts +++ b/src/packages/dumbo/src/core/schema/index.ts @@ -1,2 +1,3 @@ +export * from './components'; export * from './migrations'; export * from './schemaComponent'; diff --git a/src/packages/dumbo/src/core/schema/migrations.ts b/src/packages/dumbo/src/core/schema/migrations.ts index 2286e00a..b086508b 100644 --- a/src/packages/dumbo/src/core/schema/migrations.ts +++ b/src/packages/dumbo/src/core/schema/migrations.ts @@ -9,7 +9,7 @@ import { import { mapToCamelCase, singleOrNull } from '../query'; import { getFormatter, SQL, type SQLFormatter } from '../sql'; import { tracer } from '../tracing'; -import { schemaComponent, type SchemaComponent } from './schemaComponent'; +import { type SchemaComponent } from './schemaComponent'; export type MigrationStyle = 'None' | 'CreateOrUpdate'; @@ -32,25 +32,6 @@ export type MigrationRecord = { }; export const MIGRATIONS_LOCK_ID = 999956789; -const { AutoIncrement, Varchar, Timestamp } = SQL.column.type; - -const migrationTableSQL = SQL` - CREATE TABLE IF NOT EXISTS migrations ( - id ${AutoIncrement({ primaryKey: true })}, - name ${Varchar(255)} NOT NULL UNIQUE, - application ${Varchar(255)} NOT NULL DEFAULT 'default', - sql_hash ${Varchar(64)} NOT NULL, - timestamp ${Timestamp} NOT NULL DEFAULT CURRENT_TIMESTAMP - ); -`; - -export const migrationTableSchemaComponent = schemaComponent( - 'dumbo:schema-component:migrations-table', - { - migrations: [sqlMigration('dumbo:migrationTable:001', [migrationTableSQL])], - }, -); - declare global { var defaultMigratorOptions: Record; } diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 4d4d1112..8e9ec033 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -1,49 +1,11 @@ -import { type DatabaseDriverType, type Dumbo } from '..'; +import { SQL, type DatabaseDriverType, type Dumbo } from '..'; import { runSQLMigrations, + sqlMigration, type MigratorOptions, type SQLMigration, } from './migrations'; -export type DatabaseURNType = 'sc:dumbo:database'; -export type SchemaURNType = 'sc:dumbo:schema'; -export type TableURNType = 'sc:dumbo:table'; -export type ColumnURNType = 'sc:dumbo:column'; -export type IndexURNType = 'sc:dumbo:index'; - -export type DatabaseURN = `${DatabaseURNType}:${string}`; -export type SchemaURN = `${SchemaURNType}:${string}`; -export type TableURN = `${TableURNType}:${string}`; -export type ColumnURN = `${ColumnURNType}:${string}`; -export type IndexURN = `${IndexURNType}:${string}`; - -export const schemaComponentURN = { - database: { - type: 'sc:dumbo:database' as DatabaseURNType, - build: (name: string): DatabaseURN => `sc:dumbo:database:${name}`, - }, - schema: { - type: 'sc:dumbo:schema' as SchemaURNType, - build: (name: string): SchemaURN => `sc:dumbo:schema:${name}`, - }, - table: { - type: 'sc:dumbo:table' as TableURNType, - build: (name: string): TableURN => `sc:dumbo:table:${name}`, - }, - column: { - type: 'sc:dumbo:column' as ColumnURNType, - build: (name: string): ColumnURN => `sc:dumbo:column:${name}`, - }, - index: { - type: 'sc:dumbo:index' as IndexURNType, - build: (name: string): IndexURN => `sc:dumbo:index:${name}`, - }, - extractName: (urn: string): string => { - const parts = urn.split(':'); - return parts[parts.length - 1] || ''; - }, -} as const; - export type SchemaComponent< ComponentKey extends string = string, AdditionalData extends Record | undefined = undefined, @@ -81,6 +43,7 @@ export type SchemaComponentOptions< } & Omit; export type SchemaComponentType = `sc:${Kind}`; + export type DumboSchemaComponentType = SchemaComponentType<`dumbo:${Kind}`>; @@ -160,193 +123,24 @@ export const findSchemaComponentsOfType = ( return results; }; -export type DatabaseSchemaComponent = SchemaComponent< - DatabaseURN, - Readonly<{ - databaseName: string; - schemas: ReadonlyMap; - }> ->; - -export type DatabaseSchemaSchemaComponent = SchemaComponent< - SchemaURN, - Readonly<{ - schemaName: string; - tables: ReadonlyMap; - }> ->; - -export type TableSchemaComponent = SchemaComponent< - TableURN, - Readonly<{ - tableName: string; - columns: ReadonlyMap; - indexes: ReadonlyMap; - }> ->; - -export type ColumnSchemaComponent = SchemaComponent< - ColumnURN, - Readonly<{ - columnName: string; - }> ->; - -export type IndexSchemaComponent = SchemaComponent< - IndexURN, - Readonly<{ - indexName: string; - columns: ReadonlyMap; - isUnique: boolean; - }> ->; - -export const databaseSchemaComponent = ({ - databaseName, - schemaNames, - ...migrationsOrComponents -}: { - databaseName: string; - schemaNames?: string[]; -} & SchemaComponentOptions): DatabaseSchemaComponent => { - const schemas = - schemaNames?.map((schemaName) => - databaseSchemaSchemaComponent({ schemaName }), - ) ?? []; - - const sc = schemaComponent(schemaComponentURN.database.build(databaseName), { - migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...schemas], - }); - - return { - ...sc, - databaseName, - get schemas() { - return mapSchemaComponentsOfType( - sc.components, - schemaComponentURN.schema.type, - (c) => c.schemaName, - ); - }, - }; -}; - -export const databaseSchemaSchemaComponent = ({ - schemaName, - tableNames, - ...migrationsOrComponents -}: { - schemaName: string; - tableNames?: string[]; -} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { - const tables = - tableNames?.map((tableName) => tableSchemaComponent({ tableName })) ?? []; - - const sc = schemaComponent(schemaComponentURN.schema.build(schemaName), { - migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...tables], - }); - - return { - ...sc, - schemaName, - get tables() { - return mapSchemaComponentsOfType( - sc.components, - schemaComponentURN.table.type, - (c) => c.tableName, - ); - }, - }; -}; - -export const tableSchemaComponent = ({ - tableName, - columnNames, - ...migrationsOrComponents -}: { - tableName: string; - columnNames?: string[]; -} & SchemaComponentOptions): TableSchemaComponent => { - const columns = - columnNames?.map((columnName) => columnSchemaComponent({ columnName })) ?? - []; - - const sc = schemaComponent(schemaComponentURN.table.build(tableName), { - migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...columns], - }); +const { AutoIncrement, Varchar, Timestamp } = SQL.column.type; - return { - ...sc, - tableName, - get columns() { - return mapSchemaComponentsOfType( - sc.components, - schemaComponentURN.column.type, - (c) => c.columnName, - ); - }, - get indexes() { - return mapSchemaComponentsOfType( - sc.components, - schemaComponentURN.index.type, - (c) => c.indexName, - ); - }, - }; -}; - -export const columnSchemaComponent = ({ - columnName, - ...migrationsOrComponents -}: { - columnName: string; -} & SchemaComponentOptions): ColumnSchemaComponent => { - const sc = schemaComponent( - schemaComponentURN.column.build(columnName), - migrationsOrComponents, - ); - - return { - ...sc, - columnName, - }; -}; - -export const indexSchemaComponent = ({ - indexName, - columnNames, - isUnique, - ...migrationsOrComponents -}: { - indexName: string; - columnNames: string[]; - isUnique: boolean; -} & SchemaComponentOptions): IndexSchemaComponent => { - const columns = columnNames.map((columnName) => - columnSchemaComponent({ columnName }), +const migrationTableSQL = SQL` + CREATE TABLE IF NOT EXISTS migrations ( + id ${AutoIncrement({ primaryKey: true })}, + name ${Varchar(255)} NOT NULL UNIQUE, + application ${Varchar(255)} NOT NULL DEFAULT 'default', + sql_hash ${Varchar(64)} NOT NULL, + timestamp ${Timestamp} NOT NULL DEFAULT CURRENT_TIMESTAMP ); +`; - const sc = schemaComponent(schemaComponentURN.index.build(indexName), { - migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...columns], - }); - - return { - ...sc, - indexName, - get columns() { - return mapSchemaComponentsOfType( - sc.components, - schemaComponentURN.column.type, - (c) => c.columnName, - ); - }, - isUnique, - }; -}; +export const migrationTableSchemaComponent = schemaComponent( + 'dumbo:schema-component:migrations-table', + { + migrations: [sqlMigration('dumbo:migrationTable:001', [migrationTableSQL])], + }, +); export type SchemaComponentMigrator = { component: SchemaComponent; From 46bbdcfe90d8da0bdb395abbdd4f49302823b307 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 13:57:33 +0100 Subject: [PATCH 11/39] Refactored the structure to remove cyclic dependency between migrations and schema components --- src/packages/dumbo/src/core/schema/index.ts | 3 +- .../dumbo/src/core/schema/migrators/index.ts | 2 + .../{migrations.ts => migrators/migrator.ts} | 37 +++-------- .../migrators/schemaComponentMigrator.ts | 59 ++++++++++++++++++ .../dumbo/src/core/schema/schemaComponent.ts | 61 +------------------ .../dumbo/src/core/schema/sqlMigration.ts | 21 +++++++ 6 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 src/packages/dumbo/src/core/schema/migrators/index.ts rename src/packages/dumbo/src/core/schema/{migrations.ts => migrators/migrator.ts} (87%) create mode 100644 src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts create mode 100644 src/packages/dumbo/src/core/schema/sqlMigration.ts diff --git a/src/packages/dumbo/src/core/schema/index.ts b/src/packages/dumbo/src/core/schema/index.ts index 17b3d38d..096b665a 100644 --- a/src/packages/dumbo/src/core/schema/index.ts +++ b/src/packages/dumbo/src/core/schema/index.ts @@ -1,3 +1,4 @@ export * from './components'; -export * from './migrations'; +export * from './migrators'; export * from './schemaComponent'; +export * from './sqlMigration'; diff --git a/src/packages/dumbo/src/core/schema/migrators/index.ts b/src/packages/dumbo/src/core/schema/migrators/index.ts new file mode 100644 index 00000000..58982d62 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/migrators/index.ts @@ -0,0 +1,2 @@ +export * from './migrator'; +export * from './schemaComponentMigrator'; diff --git a/src/packages/dumbo/src/core/schema/migrations.ts b/src/packages/dumbo/src/core/schema/migrators/migrator.ts similarity index 87% rename from src/packages/dumbo/src/core/schema/migrations.ts rename to src/packages/dumbo/src/core/schema/migrators/migrator.ts index b086508b..69c8f451 100644 --- a/src/packages/dumbo/src/core/schema/migrations.ts +++ b/src/packages/dumbo/src/core/schema/migrators/migrator.ts @@ -1,35 +1,18 @@ -import type { Dumbo } from '..'; -import { type DatabaseType, fromDatabaseDriverType } from '../drivers'; -import type { SQLExecutor } from '../execute'; +import { type Dumbo } from '../..'; +import { type DatabaseType, fromDatabaseDriverType } from '../../drivers'; +import type { SQLExecutor } from '../../execute'; import { type DatabaseLock, type DatabaseLockOptions, NoDatabaseLock, -} from '../locks'; -import { mapToCamelCase, singleOrNull } from '../query'; -import { getFormatter, SQL, type SQLFormatter } from '../sql'; -import { tracer } from '../tracing'; -import { type SchemaComponent } from './schemaComponent'; +} from '../../locks'; +import { mapToCamelCase, singleOrNull } from '../../query'; +import { SQL, SQLFormatter, getFormatter } from '../../sql'; +import { tracer } from '../../tracing'; +import { type SchemaComponent } from '../schemaComponent'; +import type { MigrationRecord, SQLMigration } from '../sqlMigration'; +import { migrationTableSchemaComponent } from './schemaComponentMigrator'; -export type MigrationStyle = 'None' | 'CreateOrUpdate'; - -export type SQLMigration = { - name: string; - sqls: SQL[]; -}; - -export const sqlMigration = (name: string, sqls: SQL[]): SQLMigration => ({ - name, - sqls, -}); - -export type MigrationRecord = { - id: number; - name: string; - application: string; - sqlHash: string; - timestamp: Date; -}; export const MIGRATIONS_LOCK_ID = 999956789; declare global { diff --git a/src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts b/src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts new file mode 100644 index 00000000..1ae2db4a --- /dev/null +++ b/src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts @@ -0,0 +1,59 @@ +import type { Dumbo } from '../..'; +import type { DatabaseDriverType } from '../../drivers'; +import { SQL } from '../../sql'; +import { schemaComponent, type SchemaComponent } from '../schemaComponent'; +import { sqlMigration } from '../sqlMigration'; +import { type MigratorOptions, runSQLMigrations } from './migrator'; + +const { AutoIncrement, Varchar, Timestamp } = SQL.column.type; + +const migrationTableSQL = SQL` + CREATE TABLE IF NOT EXISTS migrations ( + id ${AutoIncrement({ primaryKey: true })}, + name ${Varchar(255)} NOT NULL UNIQUE, + application ${Varchar(255)} NOT NULL DEFAULT 'default', + sql_hash ${Varchar(64)} NOT NULL, + timestamp ${Timestamp} NOT NULL DEFAULT CURRENT_TIMESTAMP + ); +`; + +export const migrationTableSchemaComponent = schemaComponent( + 'dumbo:schema-component:migrations-table', + { + migrations: [sqlMigration('dumbo:migrationTable:001', [migrationTableSQL])], + }, +); + +export type SchemaComponentMigrator = { + component: SchemaComponent; + run: (options?: Partial) => Promise; +}; + +export const SchemaComponentMigrator = ( + component: SchemaComponent, + dumbo: Dumbo, +): SchemaComponentMigrator => { + const completedMigrations: string[] = []; + + return { + component, + run: async (options) => { + const pendingMigrations = component.migrations.filter( + (m) => + !completedMigrations.includes( + `${component.schemaComponentKey}:${m.name}`, + ), + ); + + if (pendingMigrations.length === 0) return; + + await runSQLMigrations(dumbo, pendingMigrations, options); + + completedMigrations.push( + ...pendingMigrations.map( + (m) => `${component.schemaComponentKey}:${m.name}`, + ), + ); + }, + }; +}; diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 8e9ec033..b9846f3c 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -1,10 +1,4 @@ -import { SQL, type DatabaseDriverType, type Dumbo } from '..'; -import { - runSQLMigrations, - sqlMigration, - type MigratorOptions, - type SQLMigration, -} from './migrations'; +import { type SQLMigration } from './sqlMigration'; export type SchemaComponent< ComponentKey extends string = string, @@ -122,56 +116,3 @@ export const findSchemaComponentsOfType = ( return results; }; - -const { AutoIncrement, Varchar, Timestamp } = SQL.column.type; - -const migrationTableSQL = SQL` - CREATE TABLE IF NOT EXISTS migrations ( - id ${AutoIncrement({ primaryKey: true })}, - name ${Varchar(255)} NOT NULL UNIQUE, - application ${Varchar(255)} NOT NULL DEFAULT 'default', - sql_hash ${Varchar(64)} NOT NULL, - timestamp ${Timestamp} NOT NULL DEFAULT CURRENT_TIMESTAMP - ); -`; - -export const migrationTableSchemaComponent = schemaComponent( - 'dumbo:schema-component:migrations-table', - { - migrations: [sqlMigration('dumbo:migrationTable:001', [migrationTableSQL])], - }, -); - -export type SchemaComponentMigrator = { - component: SchemaComponent; - run: (options?: Partial) => Promise; -}; - -export const SchemaComponentMigrator = ( - component: SchemaComponent, - dumbo: Dumbo, -): SchemaComponentMigrator => { - const completedMigrations: string[] = []; - - return { - component, - run: async (options) => { - const pendingMigrations = component.migrations.filter( - (m) => - !completedMigrations.includes( - `${component.schemaComponentKey}:${m.name}`, - ), - ); - - if (pendingMigrations.length === 0) return; - - await runSQLMigrations(dumbo, pendingMigrations, options); - - completedMigrations.push( - ...pendingMigrations.map( - (m) => `${component.schemaComponentKey}:${m.name}`, - ), - ); - }, - }; -}; diff --git a/src/packages/dumbo/src/core/schema/sqlMigration.ts b/src/packages/dumbo/src/core/schema/sqlMigration.ts new file mode 100644 index 00000000..590472c4 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/sqlMigration.ts @@ -0,0 +1,21 @@ +import { SQL } from '../sql'; + +export type MigrationStyle = 'None' | 'CreateOrUpdate'; + +export type SQLMigration = { + name: string; + sqls: SQL[]; +}; + +export const sqlMigration = (name: string, sqls: SQL[]): SQLMigration => ({ + name, + sqls, +}); + +export type MigrationRecord = { + id: number; + name: string; + application: string; + sqlHash: string; + timestamp: Date; +}; From a15e8541cf47c37033a0b2d978387f619dff5ca5 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 14:19:44 +0100 Subject: [PATCH 12/39] Added first version of dumbo schema --- .../components/databaseSchemaComponent.ts | 3 +- .../databaseSchemaSchemaComponent.ts | 3 +- .../schema/components/tableSchemaComponent.ts | 3 +- .../src/core/schema/dumboSchema/constants.ts | 8 + .../dumboSchema/dumboSchema.unit.spec.ts | 118 ++++++++++++ .../src/core/schema/dumboSchema/index.ts | 172 ++++++++++++++++++ src/packages/dumbo/src/core/schema/index.ts | 1 + 7 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 src/packages/dumbo/src/core/schema/dumboSchema/constants.ts create mode 100644 src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/dumboSchema/index.ts diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts index 0b032bb7..57c58ff4 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts @@ -5,6 +5,7 @@ import { type SchemaComponentOptions, } from '../schemaComponent'; import { + DatabaseSchemaURNType, databaseSchemaSchemaComponent, type DatabaseSchemaSchemaComponent, } from './databaseSchemaSchemaComponent'; @@ -48,7 +49,7 @@ export const databaseSchemaComponent = ({ get schemas() { return mapSchemaComponentsOfType( sc.components, - DatabaseURNType, + DatabaseSchemaURNType, (c) => c.schemaName, ); }, diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts index 15c51d4c..c317f3ba 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -5,6 +5,7 @@ import { type SchemaComponentOptions, } from '../schemaComponent'; import { + TableURNType, tableSchemaComponent, type TableSchemaComponent, } from './tableSchemaComponent'; @@ -49,7 +50,7 @@ export const databaseSchemaSchemaComponent = ({ get tables() { return mapSchemaComponentsOfType( sc.components, - DatabaseSchemaURNType, + TableURNType, (c) => c.tableName, ); }, diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 9a060159..8128ffda 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -5,6 +5,7 @@ import { type SchemaComponentOptions, } from '../schemaComponent'; import { + ColumnURNType, columnSchemaComponent, type ColumnSchemaComponent, } from './columnSchemaComponent'; @@ -52,7 +53,7 @@ export const tableSchemaComponent = ({ get columns() { return mapSchemaComponentsOfType( sc.components, - TableURNType, + ColumnURNType, (c) => c.columnName, ); }, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts b/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts new file mode 100644 index 00000000..720df2c8 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts @@ -0,0 +1,8 @@ +export const DEFAULT_SCHEMA = Symbol.for('dumbo.defaultSchema'); + +export const DATABASE_DEFAULTS = { + PostgreSQL: { defaultDatabase: 'postgres', defaultSchema: 'public' }, + MySQL: { defaultDatabase: null, defaultSchema: null }, + SQLite: { defaultDatabase: null, defaultSchema: 'main' }, + SqlServer: { defaultDatabase: 'master', defaultSchema: 'dbo' }, +} as const; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts new file mode 100644 index 00000000..ef8ce408 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -0,0 +1,118 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert'; +import { dumboSchema } from './index'; + +describe('dumboSchema', () => { + it('should create a column', () => { + const col = dumboSchema.column('id'); + assert.strictEqual(col.columnName, 'id'); + }); + + it('should create an index', () => { + const idx = dumboSchema.index('idx_email', ['email']); + assert.strictEqual(idx.indexName, 'idx_email'); + assert.strictEqual(idx.isUnique, false); + }); + + it('should create a unique index', () => { + const idx = dumboSchema.index('idx_email', ['email'], { unique: true }); + assert.strictEqual(idx.indexName, 'idx_email'); + assert.strictEqual(idx.isUnique, true); + }); + + it('should create a table with columns and indexes', () => { + const tbl = dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + email: dumboSchema.column('email'), + }, + indexes: { + idx_email: dumboSchema.index('idx_email', ['email']), + }, + }); + + assert.strictEqual(tbl.tableName, 'users'); + assert.strictEqual(tbl.columns.size, 2); + assert.strictEqual(tbl.indexes.size, 1); + assert.ok(tbl.columns.has('id')); + assert.ok(tbl.columns.has('email')); + assert.ok(tbl.indexes.has('idx_email')); + }); + + it('should create a named schema', () => { + const sch = dumboSchema.schema('public', { + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, + }), + }); + + assert.strictEqual(sch.schemaName, 'public'); + assert.strictEqual(sch.tables.size, 1); + assert.ok(sch.tables.has('users')); + }); + + it('should create a default schema without name', () => { + const sch = dumboSchema.schema({ + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, + }), + }); + + assert.strictEqual(sch.schemaName, ''); + assert.strictEqual(sch.tables.size, 1); + }); + + it('should create a named database', () => { + const db = dumboSchema.database('myapp', { + public: dumboSchema.schema('public', { + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, + }), + }), + }); + + assert.strictEqual(db.databaseName, 'myapp'); + assert.strictEqual(db.schemas.size, 1); + assert.ok(db.schemas.has('public')); + }); + + it('should handle DEFAULT_SCHEMA', () => { + const db = dumboSchema.database( + 'myapp', + { + [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, + }), + }), + }, + { + defaultSchemaName: 'main', + }, + ); + + assert.strictEqual(db.databaseName, 'myapp'); + assert.strictEqual(db.schemas.size, 1); + assert.ok(db.schemas.has('main')); + }); + + it('should create schema from table names', () => { + const sch = dumboSchema.schema.from('public', ['users', 'posts']); + assert.strictEqual(sch.schemaName, 'public'); + assert.strictEqual(sch.tables.size, 2); + }); + + it('should create database from schema names', () => { + const db = dumboSchema.database.from('myapp', ['public', 'analytics']); + assert.strictEqual(db.databaseName, 'myapp'); + assert.strictEqual(db.schemas.size, 2); + }); +}); diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/index.ts b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts new file mode 100644 index 00000000..b8a9dad1 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts @@ -0,0 +1,172 @@ +import { + columnSchemaComponent, + type ColumnSchemaComponent, + indexSchemaComponent, + type IndexSchemaComponent, + tableSchemaComponent, + type TableSchemaComponent, + databaseSchemaSchemaComponent, + type DatabaseSchemaSchemaComponent, + databaseSchemaComponent, + type DatabaseSchemaComponent, +} from '../components'; +import type { SchemaComponentOptions } from '../schemaComponent'; +import { DEFAULT_SCHEMA, DATABASE_DEFAULTS } from './constants'; + +const dumboColumn = ( + name: string, + options?: SchemaComponentOptions, +): ColumnSchemaComponent => + columnSchemaComponent({ + columnName: name, + ...options, + }); + +const dumboIndex = ( + name: string, + columnNames: string[], + options?: { unique?: boolean } & SchemaComponentOptions, +): IndexSchemaComponent => + indexSchemaComponent({ + indexName: name, + columnNames, + isUnique: options?.unique ?? false, + ...options, + }); + +const dumboTable = ( + name: string, + definition: { + columns?: Record; + indexes?: Record; + } & SchemaComponentOptions, +): TableSchemaComponent => { + const { columns, indexes, ...options } = definition; + + const components = [ + ...(columns ? Object.values(columns) : []), + ...(indexes ? Object.values(indexes) : []), + ]; + + return tableSchemaComponent({ + tableName: name, + components, + ...options, + }); +}; + +const dumboDatabaseSchema = ( + nameOrTables: string | Record, + tables?: Record, + options?: SchemaComponentOptions, +): DatabaseSchemaSchemaComponent => { + if (typeof nameOrTables === 'string') { + const tableComponents = Object.values(tables || {}); + return databaseSchemaSchemaComponent({ + schemaName: nameOrTables, + components: tableComponents, + ...options, + }); + } else { + const tableComponents = Object.values(nameOrTables || {}); + return databaseSchemaSchemaComponent({ + schemaName: '', + components: tableComponents, + }); + } +}; + +dumboDatabaseSchema.from = ( + schemaName: string | undefined, + tableNames: string[], +): DatabaseSchemaSchemaComponent => { + const tables = tableNames.reduce( + (acc, tableName) => { + acc[tableName] = dumboTable(tableName, {}); + return acc; + }, + {} as Record, + ); + + return schemaName + ? dumboDatabaseSchema(schemaName, tables) + : dumboDatabaseSchema(tables); +}; + +const dumboDatabase = ( + nameOrSchemas: + | string + | Record, + schemas?: Record, + options?: { defaultSchemaName?: string } & SchemaComponentOptions, +): DatabaseSchemaComponent => { + let databaseName: string; + let schemaMap: Record; + let dbOptions: typeof options; + + if (typeof nameOrSchemas === 'string') { + databaseName = nameOrSchemas; + schemaMap = schemas || {}; + dbOptions = options; + } else { + databaseName = 'database'; + schemaMap = nameOrSchemas; + dbOptions = schemas as typeof options; + } + + const schemaComponents: DatabaseSchemaSchemaComponent[] = []; + + for (const [key, schemaComponent] of Object.entries(schemaMap)) { + schemaComponents.push(schemaComponent); + } + + const symbolKeys = Object.getOwnPropertySymbols(schemaMap); + for (const key of symbolKeys) { + const schemaComponent = schemaMap[key]; + if (schemaComponent && key === DEFAULT_SCHEMA) { + const defaultSchemaName = dbOptions?.defaultSchemaName || 'public'; + schemaComponents.push( + databaseSchemaSchemaComponent({ + schemaName: defaultSchemaName, + components: Array.from(schemaComponent.components.values()), + migrations: schemaComponent.migrations, + }), + ); + } else if (schemaComponent) { + schemaComponents.push(schemaComponent); + } + } + + return databaseSchemaComponent({ + databaseName, + components: schemaComponents, + ...dbOptions, + }); +}; + +dumboDatabase.from = ( + databaseName: string | undefined, + schemaNames: string[], +): DatabaseSchemaComponent => { + const schemas = schemaNames.reduce( + (acc, schemaName) => { + acc[schemaName] = dumboDatabaseSchema(schemaName, {}); + return acc; + }, + {} as Record, + ); + + return databaseName + ? dumboDatabase(databaseName, schemas) + : dumboDatabase(schemas); +}; + +export const dumboSchema = { + database: dumboDatabase, + schema: dumboDatabaseSchema, + table: dumboTable, + column: dumboColumn, + index: dumboIndex, + DEFAULT_SCHEMA, + DATABASE_DEFAULTS, +}; diff --git a/src/packages/dumbo/src/core/schema/index.ts b/src/packages/dumbo/src/core/schema/index.ts index 096b665a..31539d0b 100644 --- a/src/packages/dumbo/src/core/schema/index.ts +++ b/src/packages/dumbo/src/core/schema/index.ts @@ -1,4 +1,5 @@ export * from './components'; +export * from './dumboSchema'; export * from './migrators'; export * from './schemaComponent'; export * from './sqlMigration'; From f87afe48bf6bf06d323b5f0ac0033d38c9a76186 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 28 Oct 2025 15:05:26 +0100 Subject: [PATCH 13/39] Added samples for dumbo schema --- plan.md | 461 ++++++++++++++++++ .../dumboSchema/dumboSchema.unit.spec.ts | 76 ++- .../src/core/schema/dumboSchema/index.ts | 12 +- 3 files changed, 531 insertions(+), 18 deletions(-) create mode 100644 plan.md diff --git a/plan.md b/plan.md new file mode 100644 index 00000000..9382dc72 --- /dev/null +++ b/plan.md @@ -0,0 +1,461 @@ +# Dumbo Database Schema Builder Implementation Plan (Simplified) + +## Overview + +Create a simple builder API for defining database schemas in dumbo, following the PongoSchema pattern but using const functions and reusing existing schema components. + +## Key Principles + +- **No overengineering** - Keep it simple +- **Use const instead of function** - Follow the existing pattern +- **Object parameters** - No function overloading, use objects with optional fields +- **No useless comments**. + +## Implementation Steps + +### Step 1: Create Constants File + +**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts` + +```typescript +// Special key for default schema +export const DEFAULT_SCHEMA = Symbol.for("dumbo.defaultSchema"); + +// Database-specific defaults (for reference/future use) +export const DATABASE_DEFAULTS = { + PostgreSQL: { defaultDatabase: "postgres", defaultSchema: "public" }, + MySQL: { defaultDatabase: null, defaultSchema: null }, + SQLite: { defaultDatabase: null, defaultSchema: "main" }, + SqlServer: { defaultDatabase: "master", defaultSchema: "dbo" }, +} as const; +``` + +**Validation**: + +- [ ] Compiles +- [ ] No linter errors + +### Step 2: Create Builder Functions + +**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/index.ts` + +```typescript +import { + columnSchemaComponent, + type ColumnSchemaComponent, + indexSchemaComponent, + type IndexSchemaComponent, + tableSchemaComponent, + type TableSchemaComponent, + databaseSchemaSchemaComponent, + type DatabaseSchemaSchemaComponent, + databaseSchemaComponent, + type DatabaseSchemaComponent, +} from "../components"; +import type { SchemaComponentOptions } from "../schemaComponent"; +import { DEFAULT_SCHEMA, DATABASE_DEFAULTS } from "./constants"; + +// Column builder - simple const +const dumboColumn = ( + name: string, + options?: SchemaComponentOptions +): ColumnSchemaComponent => + columnSchemaComponent({ + columnName: name, + ...options, + }); + +// Index builder - simple const +const dumboIndex = ( + name: string, + columnNames: string[], + options?: { unique?: boolean } & SchemaComponentOptions +): IndexSchemaComponent => + indexSchemaComponent({ + indexName: name, + columnNames, + isUnique: options?.unique ?? false, + ...options, + }); + +// Table builder - takes columns and indexes as objects +const dumboTable = ( + name: string, + definition: { + columns?: Record; + indexes?: Record; + } & SchemaComponentOptions +): TableSchemaComponent => { + const { columns, indexes, ...options } = definition; + + const components = [ + ...(columns ? Object.values(columns) : []), + ...(indexes ? Object.values(indexes) : []), + ]; + + return tableSchemaComponent({ + tableName: name, + components, + ...options, + }); +}; + +// Schema builder - name is optional for default schema +const dumboDatabaseSchema = ( + nameOrTables: string | Record, + tables?: Record, + options?: SchemaComponentOptions +): DatabaseSchemaSchemaComponent => { + if (typeof nameOrTables === "string") { + // Named schema: dumboDatabaseSchema('public', {...}) + const tableComponents = Object.values(tables || {}); + return databaseSchemaSchemaComponent({ + schemaName: nameOrTables, + components: tableComponents, + ...options, + }); + } else { + // Default schema: dumboDatabaseSchema({...}) + const tableComponents = Object.values(nameOrTables || {}); + return databaseSchemaSchemaComponent({ + schemaName: "", // Will be replaced with default + components: tableComponents, + }); + } +}; + +// Helper for creating from array of names +dumboDatabaseSchema.from = ( + schemaName: string | undefined, + tableNames: string[] +): DatabaseSchemaSchemaComponent => { + const tables = tableNames.reduce((acc, tableName) => { + acc[tableName] = dumboTable(tableName, {}); + return acc; + }, {} as Record); + + return schemaName + ? dumboDatabaseSchema(schemaName, tables) + : dumboDatabaseSchema(tables); +}; + +// Database builder - name is optional for default database +const dumboDatabase = ( + nameOrSchemas: + | string + | Record, + schemas?: Record, + options?: { defaultSchemaName?: string } & SchemaComponentOptions +): DatabaseSchemaComponent => { + let databaseName: string; + let schemaMap: Record; + let dbOptions: typeof options; + + if (typeof nameOrSchemas === "string") { + // Named database: dumboDatabase('myapp', {...}) + databaseName = nameOrSchemas; + schemaMap = schemas || {}; + dbOptions = options; + } else { + // Default database: dumboDatabase({...}) + databaseName = "database"; // Default name + schemaMap = nameOrSchemas; + dbOptions = schemas as typeof options; + } + + // Process schemas, handling DEFAULT_SCHEMA + const schemaComponents: DatabaseSchemaSchemaComponent[] = []; + + for (const [key, schemaComponent] of Object.entries(schemaMap)) { + if (key === DEFAULT_SCHEMA.toString() || key === String(DEFAULT_SCHEMA)) { + // This is the default schema - replace its name + const defaultSchemaName = dbOptions?.defaultSchemaName || "public"; + schemaComponents.push( + databaseSchemaSchemaComponent({ + schemaName: defaultSchemaName, + components: Array.from(schemaComponent.components.values()), + migrations: schemaComponent.migrations, + }) + ); + } else { + schemaComponents.push(schemaComponent); + } + } + + return databaseSchemaComponent({ + databaseName, + components: schemaComponents, + ...dbOptions, + }); +}; + +// Helper for creating from array of names +dumboDatabase.from = ( + databaseName: string | undefined, + schemaNames: string[] +): DatabaseSchemaComponent => { + const schemas = schemaNames.reduce((acc, schemaName) => { + acc[schemaName] = dumboDatabaseSchema(schemaName, {}); + return acc; + }, {} as Record); + + return databaseName + ? dumboDatabase(databaseName, schemas) + : dumboDatabase(schemas); +}; + +// Main export - similar to pongoSchema +export const dumboSchema = { + database: dumboDatabase, + schema: dumboDatabaseSchema, + table: dumboTable, + column: dumboColumn, + index: dumboIndex, + DEFAULT_SCHEMA, + DATABASE_DEFAULTS, +}; +``` + +**Reference files**: + +- `/src/packages/pongo/src/core/schema/index.ts` (pattern to follow) +- `/src/packages/dumbo/src/core/schema/components/` (components to reuse) + +**Validation**: + +- [ ] Compiles +- [ ] No linter errors +- [ ] Exports work correctly + +### Step 3: Update Main Schema Export + +**File**: `/src/packages/dumbo/src/core/schema/index.ts` + +Add to existing exports: + +```typescript +export * from "./dumboSchema"; +``` + +**Validation**: + +- [ ] Export is accessible +- [ ] No circular dependencies + +### Step 4: Create Unit Tests + +**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts` + +```typescript +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { dumboSchema } from "./index"; + +describe("dumboSchema", () => { + it("should create a column", () => { + const col = dumboSchema.column("id"); + assert.strictEqual(col.columnName, "id"); + }); + + it("should create an index", () => { + const idx = dumboSchema.index("idx_email", ["email"]); + assert.strictEqual(idx.indexName, "idx_email"); + assert.strictEqual(idx.isUnique, false); + }); + + it("should create a unique index", () => { + const idx = dumboSchema.index("idx_email", ["email"], { unique: true }); + assert.strictEqual(idx.indexName, "idx_email"); + assert.strictEqual(idx.isUnique, true); + }); + + it("should create a table with columns and indexes", () => { + const tbl = dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + email: dumboSchema.column("email"), + }, + indexes: { + idx_email: dumboSchema.index("idx_email", ["email"]), + }, + }); + + assert.strictEqual(tbl.tableName, "users"); + assert.strictEqual(tbl.columns.size, 2); + assert.strictEqual(tbl.indexes.size, 1); + assert.ok(tbl.columns.has("id")); + assert.ok(tbl.columns.has("email")); + assert.ok(tbl.indexes.has("idx_email")); + }); + + it("should create a named schema", () => { + const sch = dumboSchema.schema("public", { + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + }, + }), + }); + + assert.strictEqual(sch.schemaName, "public"); + assert.strictEqual(sch.tables.size, 1); + assert.ok(sch.tables.has("users")); + }); + + it("should create a default schema without name", () => { + const sch = dumboSchema.schema({ + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + }, + }), + }); + + assert.strictEqual(sch.schemaName, ""); + assert.strictEqual(sch.tables.size, 1); + }); + + it("should create a named database", () => { + const db = dumboSchema.database("myapp", { + public: dumboSchema.schema("public", { + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + }, + }), + }), + }); + + assert.strictEqual(db.databaseName, "myapp"); + assert.strictEqual(db.schemas.size, 1); + assert.ok(db.schemas.has("public")); + }); + + it("should handle DEFAULT_SCHEMA", () => { + const db = dumboSchema.database( + "myapp", + { + [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + }, + }), + }), + }, + { + defaultSchemaName: "main", + } + ); + + assert.strictEqual(db.databaseName, "myapp"); + assert.strictEqual(db.schemas.size, 1); + assert.ok(db.schemas.has("main")); + }); + + it("should create schema from table names", () => { + const sch = dumboSchema.schema.from("public", ["users", "posts"]); + assert.strictEqual(sch.schemaName, "public"); + assert.strictEqual(sch.tables.size, 2); + }); + + it("should create database from schema names", () => { + const db = dumboSchema.database.from("myapp", ["public", "analytics"]); + assert.strictEqual(db.databaseName, "myapp"); + assert.strictEqual(db.schemas.size, 2); + }); +}); +``` + +**Validation**: + +- [ ] All tests pass +- [ ] Tests cover main functionality + +### Step 5: Usage Example + +**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/README.md` + +```typescript +import { dumboSchema } from "@event-driven-io/dumbo"; + +// Simple database with tables in default schema +const simpleDb = dumboSchema.database( + "myapp", + { + [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + email: dumboSchema.column("email"), + name: dumboSchema.column("name"), + }, + indexes: { + idx_email: dumboSchema.index("idx_email", ["email"], { + unique: true, + }), + }, + }), + }), + }, + { + defaultSchemaName: "public", // PostgreSQL default + } +); + +// Database with multiple schemas +const multiSchemaDb = dumboSchema.database("myapp", { + public: dumboSchema.schema("public", { + users: dumboSchema.table("users", { + columns: { + id: dumboSchema.column("id"), + email: dumboSchema.column("email"), + }, + }), + }), + analytics: dumboSchema.schema("analytics", { + events: dumboSchema.table("events", { + columns: { + id: dumboSchema.column("id"), + user_id: dumboSchema.column("user_id"), + timestamp: dumboSchema.column("timestamp"), + }, + }), + }), +}); + +// Access using name-based maps +const publicSchema = multiSchemaDb.schemas.get("public"); +const usersTable = publicSchema?.tables.get("users"); +const emailColumn = usersTable?.columns.get("email"); +``` + +## Execution Plan + +### Setup Subagents + +1. **Build Agent**: `npm run build:ts:watch` from `/home/oskar/Repos/Pongo/src` +2. **Linter Agent**: `npm run fix` on demand from `/home/oskar/Repos/Pongo/src` +3. **Test Agent**: `npm run test:unit` from `/home/oskar/Repos/Pongo/src` + +### Implementation Process + +1. Start all subagents +2. Create constants.ts → Check compilation → Run linter → **Ask for confirmation** ✅ +3. Create index.ts → Check compilation → Run linter → **Ask for confirmation** ✅ +4. Update exports → Check compilation → Run linter → **Ask for confirmation** ✅ +5. Create tests → Ensure tests pass → **Ask for confirmation** ✅ +6. Don't commit after step. User will do it when confirming. + +## Success Criteria + +1. ✅ Code compiles +2. ✅ Linter passes +3. ✅ Unit tests pass +4. ✅ Fix issues, don't just report them + +## Questions to Ask If Unsure + +- Should column/index builders accept additional metadata beyond what's in the current components? Not for now. +- Should we validate that DEFAULT_SCHEMA isn't used as an actual schema name? Yes. +- Should the default database name be configurable or just use 'database'? Same as schema diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index ef8ce408..f71cc721 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -1,26 +1,26 @@ -import { describe, it } from 'node:test'; import assert from 'node:assert'; +import { describe, it } from 'node:test'; import { dumboSchema } from './index'; -describe('dumboSchema', () => { - it('should create a column', () => { +void describe('dumboSchema', () => { + void it('should create a column', () => { const col = dumboSchema.column('id'); assert.strictEqual(col.columnName, 'id'); }); - it('should create an index', () => { + void it('should create an index', () => { const idx = dumboSchema.index('idx_email', ['email']); assert.strictEqual(idx.indexName, 'idx_email'); assert.strictEqual(idx.isUnique, false); }); - it('should create a unique index', () => { + void it('should create a unique index', () => { const idx = dumboSchema.index('idx_email', ['email'], { unique: true }); assert.strictEqual(idx.indexName, 'idx_email'); assert.strictEqual(idx.isUnique, true); }); - it('should create a table with columns and indexes', () => { + void it('should create a table with columns and indexes', () => { const tbl = dumboSchema.table('users', { columns: { id: dumboSchema.column('id'), @@ -39,7 +39,7 @@ describe('dumboSchema', () => { assert.ok(tbl.indexes.has('idx_email')); }); - it('should create a named schema', () => { + void it('should create a named schema', () => { const sch = dumboSchema.schema('public', { users: dumboSchema.table('users', { columns: { @@ -53,7 +53,7 @@ describe('dumboSchema', () => { assert.ok(sch.tables.has('users')); }); - it('should create a default schema without name', () => { + void it('should create a default schema without name', () => { const sch = dumboSchema.schema({ users: dumboSchema.table('users', { columns: { @@ -66,7 +66,7 @@ describe('dumboSchema', () => { assert.strictEqual(sch.tables.size, 1); }); - it('should create a named database', () => { + void it('should create a named database', () => { const db = dumboSchema.database('myapp', { public: dumboSchema.schema('public', { users: dumboSchema.table('users', { @@ -82,7 +82,7 @@ describe('dumboSchema', () => { assert.ok(db.schemas.has('public')); }); - it('should handle DEFAULT_SCHEMA', () => { + void it('should handle DEFAULT_SCHEMA', () => { const db = dumboSchema.database( 'myapp', { @@ -104,15 +104,67 @@ describe('dumboSchema', () => { assert.ok(db.schemas.has('main')); }); - it('should create schema from table names', () => { + void it('should create schema from table names', () => { const sch = dumboSchema.schema.from('public', ['users', 'posts']); assert.strictEqual(sch.schemaName, 'public'); assert.strictEqual(sch.tables.size, 2); }); - it('should create database from schema names', () => { + void it('should create database from schema names', () => { const db = dumboSchema.database.from('myapp', ['public', 'analytics']); assert.strictEqual(db.databaseName, 'myapp'); assert.strictEqual(db.schemas.size, 2); }); }); + +// Samples + +// Simple database with tables in default schema +export const simpleDb = dumboSchema.database( + 'myapp', + { + [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + email: dumboSchema.column('email'), + name: dumboSchema.column('name'), + }, + indexes: { + idx_email: dumboSchema.index('idx_email', ['email'], { + unique: true, + }), + }, + }), + }), + }, + { + defaultSchemaName: 'public', // PostgreSQL default + }, +); + +// Database with multiple schemas +const multiSchemaDb = dumboSchema.database('myapp', { + public: dumboSchema.schema('public', { + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + email: dumboSchema.column('email'), + }, + }), + }), + analytics: dumboSchema.schema('analytics', { + events: dumboSchema.table('events', { + columns: { + id: dumboSchema.column('id'), + user_id: dumboSchema.column('user_id'), + timestamp: dumboSchema.column('timestamp'), + }, + }), + }), +}); + +// Access using name-based maps +const publicSchema = multiSchemaDb.schemas.get('public'); +const usersTable = publicSchema?.tables.get('users'); +export const emailColumn = usersTable?.columns.get('email'); diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/index.ts b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts index b8a9dad1..f8b8d12d 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/index.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts @@ -1,17 +1,17 @@ import { columnSchemaComponent, type ColumnSchemaComponent, + databaseSchemaComponent, + type DatabaseSchemaComponent, + databaseSchemaSchemaComponent, + type DatabaseSchemaSchemaComponent, indexSchemaComponent, type IndexSchemaComponent, tableSchemaComponent, type TableSchemaComponent, - databaseSchemaSchemaComponent, - type DatabaseSchemaSchemaComponent, - databaseSchemaComponent, - type DatabaseSchemaComponent, } from '../components'; import type { SchemaComponentOptions } from '../schemaComponent'; -import { DEFAULT_SCHEMA, DATABASE_DEFAULTS } from './constants'; +import { DATABASE_DEFAULTS, DEFAULT_SCHEMA } from './constants'; const dumboColumn = ( name: string, @@ -116,7 +116,7 @@ const dumboDatabase = ( const schemaComponents: DatabaseSchemaSchemaComponent[] = []; - for (const [key, schemaComponent] of Object.entries(schemaMap)) { + for (const [_, schemaComponent] of Object.entries(schemaMap)) { schemaComponents.push(schemaComponent); } From 20a2d9234e14580b2bda1c491cbbbb8c3f68f5ec Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 29 Oct 2025 19:29:09 +0100 Subject: [PATCH 14/39] Simplified database schema setup --- .../databaseSchemaSchemaComponent.ts | 5 +- .../src/core/schema/dumboSchema/constants.ts | 8 - .../core/schema/dumboSchema/dumboSchema.ts | 176 ++++++++++++++++++ .../dumboSchema/dumboSchema.unit.spec.ts | 68 ++++--- .../src/core/schema/dumboSchema/index.ts | 173 +---------------- 5 files changed, 217 insertions(+), 213 deletions(-) delete mode 100644 src/packages/dumbo/src/core/schema/dumboSchema/constants.ts create mode 100644 src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts index c317f3ba..cd39ce4e 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -10,10 +10,11 @@ import { type TableSchemaComponent, } from './tableSchemaComponent'; -export type DatabaseSchemaURNType = 'sc:dumbo:schema'; +export type DatabaseSchemaURNType = 'sc:dumbo:database_schema'; export type DatabaseSchemaURN = `${DatabaseSchemaURNType}:${string}`; -export const DatabaseSchemaURNType: DatabaseSchemaURNType = 'sc:dumbo:schema'; +export const DatabaseSchemaURNType: DatabaseSchemaURNType = + 'sc:dumbo:database_schema'; export const DatabaseSchemaURN = ({ name, }: { diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts b/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts deleted file mode 100644 index 720df2c8..00000000 --- a/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const DEFAULT_SCHEMA = Symbol.for('dumbo.defaultSchema'); - -export const DATABASE_DEFAULTS = { - PostgreSQL: { defaultDatabase: 'postgres', defaultSchema: 'public' }, - MySQL: { defaultDatabase: null, defaultSchema: null }, - SQLite: { defaultDatabase: null, defaultSchema: 'main' }, - SqlServer: { defaultDatabase: 'master', defaultSchema: 'dbo' }, -} as const; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts new file mode 100644 index 00000000..9435db96 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -0,0 +1,176 @@ +import { + columnSchemaComponent, + type ColumnSchemaComponent, + databaseSchemaComponent, + type DatabaseSchemaComponent, + databaseSchemaSchemaComponent, + type DatabaseSchemaSchemaComponent, + indexSchemaComponent, + type IndexSchemaComponent, + tableSchemaComponent, + type TableSchemaComponent, +} from '../components'; +import { + type AnySchemaComponent, + isSchemaComponentOfType, + type SchemaComponentOptions, +} from '../schemaComponent'; + +const DEFAULT_DATABASE_NAME = '__default_database__'; +const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; + +const dumboColumn = ( + name: string, + options?: SchemaComponentOptions, +): ColumnSchemaComponent => + columnSchemaComponent({ + columnName: name, + ...options, + }); + +const dumboIndex = ( + name: string, + columnNames: string[], + options?: { unique?: boolean } & SchemaComponentOptions, +): IndexSchemaComponent => + indexSchemaComponent({ + indexName: name, + columnNames, + isUnique: options?.unique ?? false, + ...options, + }); + +const dumboTable = ( + name: string, + definition: { + columns?: Record; + indexes?: Record; + } & SchemaComponentOptions, +): TableSchemaComponent => { + const { columns, indexes, ...options } = definition; + + const components = [ + ...(columns ? Object.values(columns) : []), + ...(indexes ? Object.values(indexes) : []), + ]; + + return tableSchemaComponent({ + tableName: name, + components, + ...options, + }); +}; + +const dumboDatabaseSchema = ( + nameOrTables: string | Record, + tables?: Record, + options?: SchemaComponentOptions, +): DatabaseSchemaSchemaComponent => { + if (typeof nameOrTables === 'string') { + const tableComponents = Object.values(tables || {}); + return databaseSchemaSchemaComponent({ + schemaName: nameOrTables, + components: tableComponents, + ...options, + }); + } else { + const tableComponents = Object.values(nameOrTables || {}); + return databaseSchemaSchemaComponent({ + schemaName: DEFAULT_DATABASE_SCHEMA_NAME, + components: tableComponents, + }); + } +}; + +dumboDatabaseSchema.from = ( + schemaName: string | undefined, + tableNames: string[], +): DatabaseSchemaSchemaComponent => { + const tables = tableNames.reduce( + (acc, tableName) => { + acc[tableName] = dumboTable(tableName, {}); + return acc; + }, + {} as Record, + ); + + return schemaName + ? dumboDatabaseSchema(schemaName, tables) + : dumboDatabaseSchema(tables); +}; + +const dumboDatabase = ( + nameOrSchemas: + | string + | DatabaseSchemaSchemaComponent + | Record, + schemasOrOptions?: + | DatabaseSchemaSchemaComponent + | Record + | SchemaComponentOptions, + options?: SchemaComponentOptions, +): DatabaseSchemaComponent => { + const databaseName = + typeof nameOrSchemas === 'string' ? nameOrSchemas : DEFAULT_DATABASE_NAME; + + const schemasOrSchema = + typeof nameOrSchemas === 'string' + ? (schemasOrOptions ?? {}) + : nameOrSchemas; + const schemaMap: Record = + 'schemaComponentKey' in schemasOrSchema && + isSchemaComponentOfType( + schemasOrSchema as AnySchemaComponent, + 'sc:dumbo:database_schema', + ) + ? { + [DEFAULT_DATABASE_SCHEMA_NAME]: + schemasOrSchema as DatabaseSchemaSchemaComponent, + } + : (schemasOrSchema as Record); + + const dbOptions: typeof options = + typeof nameOrSchemas === 'string' + ? options + : (schemasOrOptions as typeof options); + + const schemaComponents: DatabaseSchemaSchemaComponent[] = []; + + for (const [_, schemaComponent] of Object.entries(schemaMap)) { + schemaComponents.push(schemaComponent); + } + + return databaseSchemaComponent({ + databaseName, + components: schemaComponents, + ...dbOptions, + }); +}; + +dumboDatabase.from = ( + databaseName: string | undefined, + schemaNames: string[], +): DatabaseSchemaComponent => { + const schemas = schemaNames.reduce( + (acc, schemaName) => { + acc[schemaName] = dumboDatabaseSchema(schemaName, {}); + return acc; + }, + {} as Record, + ); + + return databaseName + ? dumboDatabase(databaseName, schemas) + : dumboDatabase(schemas); +}; + +dumboDatabase.defaultName = DEFAULT_DATABASE_NAME; +dumboDatabaseSchema.defaultName = DEFAULT_DATABASE_SCHEMA_NAME; + +export const dumboSchema = { + database: dumboDatabase, + schema: dumboDatabaseSchema, + table: dumboTable, + column: dumboColumn, + index: dumboIndex, +}; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index f71cc721..77451815 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -62,10 +62,26 @@ void describe('dumboSchema', () => { }), }); - assert.strictEqual(sch.schemaName, ''); + assert.strictEqual(sch.schemaName, dumboSchema.schema.defaultName); assert.strictEqual(sch.tables.size, 1); }); + void it('should create a default database', () => { + const db = dumboSchema.database({ + public: dumboSchema.schema('public', { + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, + }), + }), + }); + + assert.strictEqual(db.databaseName, dumboSchema.database.defaultName); + assert.strictEqual(db.schemas.size, 1); + assert.ok(db.schemas.has('public')); + }); + void it('should create a named database', () => { const db = dumboSchema.database('myapp', { public: dumboSchema.schema('public', { @@ -85,23 +101,18 @@ void describe('dumboSchema', () => { void it('should handle DEFAULT_SCHEMA', () => { const db = dumboSchema.database( 'myapp', - { - [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ - users: dumboSchema.table('users', { - columns: { - id: dumboSchema.column('id'), - }, - }), + dumboSchema.schema({ + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + }, }), - }, - { - defaultSchemaName: 'main', - }, + }), ); assert.strictEqual(db.databaseName, 'myapp'); assert.strictEqual(db.schemas.size, 1); - assert.ok(db.schemas.has('main')); + assert.ok(db.schemas.has(dumboSchema.schema.defaultName)); }); void it('should create schema from table names', () => { @@ -122,25 +133,20 @@ void describe('dumboSchema', () => { // Simple database with tables in default schema export const simpleDb = dumboSchema.database( 'myapp', - { - [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ - users: dumboSchema.table('users', { - columns: { - id: dumboSchema.column('id'), - email: dumboSchema.column('email'), - name: dumboSchema.column('name'), - }, - indexes: { - idx_email: dumboSchema.index('idx_email', ['email'], { - unique: true, - }), - }, - }), + dumboSchema.schema({ + users: dumboSchema.table('users', { + columns: { + id: dumboSchema.column('id'), + email: dumboSchema.column('email'), + name: dumboSchema.column('name'), + }, + indexes: { + idx_email: dumboSchema.index('idx_email', ['email'], { + unique: true, + }), + }, }), - }, - { - defaultSchemaName: 'public', // PostgreSQL default - }, + }), ); // Database with multiple schemas diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/index.ts b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts index f8b8d12d..5ad75ea3 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/index.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/index.ts @@ -1,172 +1 @@ -import { - columnSchemaComponent, - type ColumnSchemaComponent, - databaseSchemaComponent, - type DatabaseSchemaComponent, - databaseSchemaSchemaComponent, - type DatabaseSchemaSchemaComponent, - indexSchemaComponent, - type IndexSchemaComponent, - tableSchemaComponent, - type TableSchemaComponent, -} from '../components'; -import type { SchemaComponentOptions } from '../schemaComponent'; -import { DATABASE_DEFAULTS, DEFAULT_SCHEMA } from './constants'; - -const dumboColumn = ( - name: string, - options?: SchemaComponentOptions, -): ColumnSchemaComponent => - columnSchemaComponent({ - columnName: name, - ...options, - }); - -const dumboIndex = ( - name: string, - columnNames: string[], - options?: { unique?: boolean } & SchemaComponentOptions, -): IndexSchemaComponent => - indexSchemaComponent({ - indexName: name, - columnNames, - isUnique: options?.unique ?? false, - ...options, - }); - -const dumboTable = ( - name: string, - definition: { - columns?: Record; - indexes?: Record; - } & SchemaComponentOptions, -): TableSchemaComponent => { - const { columns, indexes, ...options } = definition; - - const components = [ - ...(columns ? Object.values(columns) : []), - ...(indexes ? Object.values(indexes) : []), - ]; - - return tableSchemaComponent({ - tableName: name, - components, - ...options, - }); -}; - -const dumboDatabaseSchema = ( - nameOrTables: string | Record, - tables?: Record, - options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent => { - if (typeof nameOrTables === 'string') { - const tableComponents = Object.values(tables || {}); - return databaseSchemaSchemaComponent({ - schemaName: nameOrTables, - components: tableComponents, - ...options, - }); - } else { - const tableComponents = Object.values(nameOrTables || {}); - return databaseSchemaSchemaComponent({ - schemaName: '', - components: tableComponents, - }); - } -}; - -dumboDatabaseSchema.from = ( - schemaName: string | undefined, - tableNames: string[], -): DatabaseSchemaSchemaComponent => { - const tables = tableNames.reduce( - (acc, tableName) => { - acc[tableName] = dumboTable(tableName, {}); - return acc; - }, - {} as Record, - ); - - return schemaName - ? dumboDatabaseSchema(schemaName, tables) - : dumboDatabaseSchema(tables); -}; - -const dumboDatabase = ( - nameOrSchemas: - | string - | Record, - schemas?: Record, - options?: { defaultSchemaName?: string } & SchemaComponentOptions, -): DatabaseSchemaComponent => { - let databaseName: string; - let schemaMap: Record; - let dbOptions: typeof options; - - if (typeof nameOrSchemas === 'string') { - databaseName = nameOrSchemas; - schemaMap = schemas || {}; - dbOptions = options; - } else { - databaseName = 'database'; - schemaMap = nameOrSchemas; - dbOptions = schemas as typeof options; - } - - const schemaComponents: DatabaseSchemaSchemaComponent[] = []; - - for (const [_, schemaComponent] of Object.entries(schemaMap)) { - schemaComponents.push(schemaComponent); - } - - const symbolKeys = Object.getOwnPropertySymbols(schemaMap); - for (const key of symbolKeys) { - const schemaComponent = schemaMap[key]; - if (schemaComponent && key === DEFAULT_SCHEMA) { - const defaultSchemaName = dbOptions?.defaultSchemaName || 'public'; - schemaComponents.push( - databaseSchemaSchemaComponent({ - schemaName: defaultSchemaName, - components: Array.from(schemaComponent.components.values()), - migrations: schemaComponent.migrations, - }), - ); - } else if (schemaComponent) { - schemaComponents.push(schemaComponent); - } - } - - return databaseSchemaComponent({ - databaseName, - components: schemaComponents, - ...dbOptions, - }); -}; - -dumboDatabase.from = ( - databaseName: string | undefined, - schemaNames: string[], -): DatabaseSchemaComponent => { - const schemas = schemaNames.reduce( - (acc, schemaName) => { - acc[schemaName] = dumboDatabaseSchema(schemaName, {}); - return acc; - }, - {} as Record, - ); - - return databaseName - ? dumboDatabase(databaseName, schemas) - : dumboDatabase(schemas); -}; - -export const dumboSchema = { - database: dumboDatabase, - schema: dumboDatabaseSchema, - table: dumboTable, - column: dumboColumn, - index: dumboIndex, - DEFAULT_SCHEMA, - DATABASE_DEFAULTS, -}; +export * from './dumboSchema'; From 8ff99b32d91892d09466c04b3eb72408998d4451 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 30 Oct 2025 09:36:47 +0100 Subject: [PATCH 15/39] Addef dumbo schema functions overloads --- .../core/schema/dumboSchema/dumboSchema.ts | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 9435db96..0b5885cb 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -61,26 +61,33 @@ const dumboTable = ( }); }; -const dumboDatabaseSchema = ( +function dumboDatabaseSchema( + tables: Record, +): DatabaseSchemaSchemaComponent; +function dumboDatabaseSchema( + schemaName: string, + tables: Record, + options?: SchemaComponentOptions, +): DatabaseSchemaSchemaComponent; +function dumboDatabaseSchema( nameOrTables: string | Record, tables?: Record, options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent => { - if (typeof nameOrTables === 'string') { - const tableComponents = Object.values(tables || {}); - return databaseSchemaSchemaComponent({ - schemaName: nameOrTables, - components: tableComponents, - ...options, - }); - } else { - const tableComponents = Object.values(nameOrTables || {}); - return databaseSchemaSchemaComponent({ - schemaName: DEFAULT_DATABASE_SCHEMA_NAME, - components: tableComponents, - }); - } -}; +): DatabaseSchemaSchemaComponent { + const schemaName = + typeof nameOrTables === 'string' + ? nameOrTables + : DEFAULT_DATABASE_SCHEMA_NAME; + const components = Object.values( + (typeof nameOrTables === 'string' ? tables : nameOrTables) ?? {}, + ); + + return databaseSchemaSchemaComponent({ + schemaName, + components, + ...options, + }); +} dumboDatabaseSchema.from = ( schemaName: string | undefined, @@ -99,7 +106,23 @@ dumboDatabaseSchema.from = ( : dumboDatabaseSchema(tables); }; -const dumboDatabase = ( +function dumboDatabase( + schemas: Record, +): DatabaseSchemaComponent; +function dumboDatabase( + schema: DatabaseSchemaSchemaComponent, +): DatabaseSchemaComponent; +function dumboDatabase( + databaseName: string, + schemas: Record, + options?: SchemaComponentOptions, +): DatabaseSchemaComponent; +function dumboDatabase( + databaseName: string, + schema: DatabaseSchemaSchemaComponent, + options?: SchemaComponentOptions, +): DatabaseSchemaComponent; +function dumboDatabase( nameOrSchemas: | string | DatabaseSchemaSchemaComponent @@ -109,7 +132,7 @@ const dumboDatabase = ( | Record | SchemaComponentOptions, options?: SchemaComponentOptions, -): DatabaseSchemaComponent => { +): DatabaseSchemaComponent { const databaseName = typeof nameOrSchemas === 'string' ? nameOrSchemas : DEFAULT_DATABASE_NAME; @@ -145,7 +168,7 @@ const dumboDatabase = ( components: schemaComponents, ...dbOptions, }); -}; +} dumboDatabase.from = ( databaseName: string | undefined, From 03fb40ad7b7f217f50d023fe84a8325d11b01571 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 30 Oct 2025 09:51:28 +0100 Subject: [PATCH 16/39] Refactored Schema component to use exclude instead of omit to avoid accidental overrides --- .../dumbo/src/core/schema/schemaComponent.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index b9846f3c..7db1e3c1 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -2,7 +2,16 @@ import { type SQLMigration } from './sqlMigration'; export type SchemaComponent< ComponentKey extends string = string, - AdditionalData extends Record | undefined = undefined, + AdditionalData extends + | Exclude< + Record, + | 'schemaComponentKey' + | 'components' + | 'migrations' + | 'addComponent' + | 'addMigration' + > + | undefined = undefined, > = { schemaComponentKey: ComponentKey; components: ReadonlyMap; @@ -10,7 +19,7 @@ export type SchemaComponent< // eslint-disable-next-line @typescript-eslint/no-explicit-any addComponent: (component: SchemaComponent) => void; addMigration: (migration: SQLMigration) => void; -} & Omit< +} & Exclude< // eslint-disable-next-line @typescript-eslint/no-empty-object-type AdditionalData extends undefined ? {} : AdditionalData, | 'schemaComponentKey' From 16fde917d6787980961bbc82ebb04be00fe2271f Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Fri, 31 Oct 2025 13:27:33 +0100 Subject: [PATCH 17/39] Added explicit methods for adding nested components --- .../components/databaseSchemaComponent.ts | 15 ++++++++-- .../databaseSchemaSchemaComponent.ts | 13 +++++++-- .../schema/components/indexSchemaComponent.ts | 9 ++++++ .../schema/components/tableSchemaComponent.ts | 19 ++++++++++--- .../dumbo/src/core/schema/schemaComponent.ts | 28 +++++++++++++++---- 5 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts index 57c58ff4..0f6da357 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts @@ -22,6 +22,9 @@ export type DatabaseSchemaComponent = SchemaComponent< Readonly<{ databaseName: string; schemas: ReadonlyMap; + addSchema: ( + schema: string | DatabaseSchemaSchemaComponent, + ) => DatabaseSchemaSchemaComponent; }> >; @@ -38,20 +41,26 @@ export const databaseSchemaComponent = ({ databaseSchemaSchemaComponent({ schemaName }), ) ?? []; - const sc = schemaComponent(DatabaseURN({ name: databaseName }), { + const base = schemaComponent(DatabaseURN({ name: databaseName }), { migrations: migrationsOrComponents.migrations ?? [], components: [...(migrationsOrComponents.components ?? []), ...schemas], }); return { - ...sc, + ...base, databaseName, get schemas() { return mapSchemaComponentsOfType( - sc.components, + base.components, DatabaseSchemaURNType, (c) => c.schemaName, ); }, + addSchema: (schema: string | DatabaseSchemaSchemaComponent) => + base.addComponent( + typeof schema === 'string' + ? databaseSchemaSchemaComponent({ schemaName: schema }) + : schema, + ), }; }; diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts index cd39ce4e..20bdafdc 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -26,6 +26,7 @@ export type DatabaseSchemaSchemaComponent = SchemaComponent< Readonly<{ schemaName: string; tables: ReadonlyMap; + addTable: (table: string | TableSchemaComponent) => TableSchemaComponent; }> >; @@ -40,20 +41,26 @@ export const databaseSchemaSchemaComponent = ({ const tables = tableNames?.map((tableName) => tableSchemaComponent({ tableName })) ?? []; - const sc = schemaComponent(DatabaseSchemaURN({ name: schemaName }), { + const base = schemaComponent(DatabaseSchemaURN({ name: schemaName }), { migrations: migrationsOrComponents.migrations ?? [], components: [...(migrationsOrComponents.components ?? []), ...tables], }); return { - ...sc, + ...base, schemaName, get tables() { return mapSchemaComponentsOfType( - sc.components, + base.components, TableURNType, (c) => c.tableName, ); }, + addTable: (table: string | TableSchemaComponent) => + base.addComponent( + typeof table === 'string' + ? tableSchemaComponent({ tableName: table }) + : table, + ), }; }; diff --git a/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts index dbedd9a1..8508c8a9 100644 --- a/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts @@ -18,6 +18,9 @@ export type IndexSchemaComponent = SchemaComponent< indexName: string; columns: ReadonlyMap; isUnique: boolean; + addColumn: ( + column: string | ColumnSchemaComponent, + ) => ColumnSchemaComponent; }> >; @@ -54,6 +57,12 @@ export const indexSchemaComponent = ({ (c) => c.columnName, ); }, + addColumn: (column: string | ColumnSchemaComponent) => + sc.addComponent( + typeof column === 'string' + ? columnSchemaComponent({ columnName: column }) + : column, + ), isUnique, }; }; diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 8128ffda..300e31c3 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -27,6 +27,10 @@ export type TableSchemaComponent = SchemaComponent< tableName: string; columns: ReadonlyMap; indexes: ReadonlyMap; + addColumn: ( + column: string | ColumnSchemaComponent, + ) => ColumnSchemaComponent; + addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; }> >; @@ -42,27 +46,34 @@ export const tableSchemaComponent = ({ columnNames?.map((columnName) => columnSchemaComponent({ columnName })) ?? []; - const sc = schemaComponent(TableURN({ name: tableName }), { + const base = schemaComponent(TableURN({ name: tableName }), { migrations: migrationsOrComponents.migrations ?? [], components: [...(migrationsOrComponents.components ?? []), ...columns], }); return { - ...sc, + ...base, tableName, get columns() { return mapSchemaComponentsOfType( - sc.components, + base.components, ColumnURNType, (c) => c.columnName, ); }, get indexes() { return mapSchemaComponentsOfType( - sc.components, + base.components, IndexURNType, (c) => c.indexName, ); }, + addColumn: (column: string | ColumnSchemaComponent) => + base.addComponent( + typeof column === 'string' + ? columnSchemaComponent({ columnName: column }) + : column, + ), + addIndex: (index: IndexSchemaComponent) => base.addComponent(index), }; }; diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 7db1e3c1..78d4b713 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -16,8 +16,20 @@ export type SchemaComponent< schemaComponentKey: ComponentKey; components: ReadonlyMap; migrations: ReadonlyArray; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - addComponent: (component: SchemaComponent) => void; + + addComponent: < + SchemaComponentType extends SchemaComponent< + string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Record + > = SchemaComponent< + string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Record + >, + >( + component: SchemaComponentType, + ) => SchemaComponentType; addMigration: (migration: SQLMigration) => void; } & Exclude< // eslint-disable-next-line @typescript-eslint/no-empty-object-type @@ -33,7 +45,8 @@ export type ExtractAdditionalData = T extends SchemaComponent ? Data : never; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnySchemaComponent = SchemaComponent; +export type AnySchemaComponent = SchemaComponent>; + export type AnySchemaComponentOfType = // eslint-disable-next-line @typescript-eslint/no-explicit-any SchemaComponent; @@ -69,9 +82,14 @@ export const schemaComponent = ( ...Array.from(componentsMap.values()).flatMap((c) => c.migrations), ]; }, - addComponent: (component: SchemaComponent) => { + addComponent: < + SchemaComponentType extends AnySchemaComponent = AnySchemaComponent, + >( + component: SchemaComponentType, + ): SchemaComponentType => { componentsMap.set(component.schemaComponentKey, component); migrations.push(...component.migrations); + return component; }, addMigration: (migration: SQLMigration) => { migrations.push(migration); @@ -101,7 +119,7 @@ export const mapSchemaComponentsOfType = ( Array.from(components.entries()) .filter(([urn]) => urn.startsWith(prefix)) .map(([urn, component]) => [ - keyMapper ? keyMapper(component) : urn, + keyMapper ? keyMapper(component as T) : urn, component as T, ]), ); From 70aafb137ca6b7df97627c4654c5054ac2a07200 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Fri, 31 Oct 2025 18:32:13 +0100 Subject: [PATCH 18/39] Added design and implementation plan --- .../core/schema/MIGRATION_UNIFICATION_PLAN.md | 1597 +++++++++++++++++ src/packages/dumbo/src/core/schema/REVIEW.MD | 201 +++ 2 files changed, 1798 insertions(+) create mode 100644 src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md create mode 100644 src/packages/dumbo/src/core/schema/REVIEW.MD diff --git a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md new file mode 100644 index 00000000..c867b058 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md @@ -0,0 +1,1597 @@ +# Schema Component Migration Unification Plan + +## Overview +This plan details the unification of SQL migrations and TypeScript schema definitions in the Dumbo schema component system. The goal is to support both SQL-first and TypeScript-first workflows while maintaining a clear source of truth and enabling strongly-typed query builders for Pongo. + +## Context and Current State + +### Key Files +- **Core Schema Component**: [src/packages/dumbo/src/core/schema/schemaComponent.ts](src/packages/dumbo/src/core/schema/schemaComponent.ts) - Lines 3-146 +- **Component Implementations**: + - [src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts) + - [src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts) + - [src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts) + - [src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts) + - [src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts) +- **Migration System**: + - [src/packages/dumbo/src/core/schema/sqlMigration.ts](src/packages/dumbo/src/core/schema/sqlMigration.ts) + - [src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts](src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts) + - [src/packages/dumbo/src/core/schema/migrators/migrator.ts](src/packages/dumbo/src/core/schema/migrators/migrator.ts) +- **Builder API**: [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts) +- **Tests**: [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts) +- **Pongo Integration Target**: [src/packages/pongo/src/core/schema/index.ts](src/packages/pongo/src/core/schema/index.ts) + +### Current Problem +The system cannot distinguish between: +1. **Initial schema definitions** (CREATE TABLE) +2. **Schema evolution** (ALTER TABLE) +3. **Data migrations** (UPDATE/INSERT) + +All migrations are stored in a single `migrations` array and bubble up through the component hierarchy (lines 79-83 in schemaComponent.ts), making it impossible to know the intent or current state. + +## Agreed Solution + +### Core Principle: TypeScript as Source of Truth (When Present) + +**The Rule**: When TypeScript definitions exist, they represent the desired schema state. Migrations are either: +1. **Generated** from TypeScript definitions (if no migrations provided) +2. **Validated** against TypeScript definitions (if migrations provided) +3. **Trusted** completely (if no TypeScript definitions provided - SQL-first mode) + +### Three Usage Modes + +#### 1. TypeScript-First (Generates Migrations) +```typescript +const table = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255 }) + } + // No migrations = will generate CREATE TABLE IF NOT EXISTS +}); +``` + +#### 2. SQL-First (Migrations as Source of Truth) +```typescript +const table = tableSchemaComponent({ + tableName: 'users', + migrations: [ + sqlMigration('001_create', [SQL`CREATE TABLE users ...`]) + ] + // No columns = trust the SQL completely +}); +``` + +#### 3. Hybrid with Validation +```typescript +const table = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial' }), + email: column('email', { type: 'varchar' }) + }, + migrations: [ + sqlMigration('001_create', [SQL`CREATE TABLE users ...`]) + ] + // Both provided = TypeScript is truth, validate migrations match +}); +``` + +## Implementation Plan + +### Phase 1: Enhanced Column Definition Types + +**File to modify**: [src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts) + +```typescript +export type ColumnDefinition = { + type: 'serial' | 'varchar' | 'int' | 'bigint' | 'text' | 'timestamp' | 'boolean' | 'uuid' | 'json' | 'jsonb'; + nullable?: boolean; + primaryKey?: boolean; + unique?: boolean; + length?: number; + precision?: number; + scale?: number; + defaultValue?: unknown; + references?: { + table: string; + column: string; + onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; + onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; + }; +}; + +export type ColumnSchemaComponent = SchemaComponent< + ColumnURN, + Readonly<{ + columnName: string; + definition?: ColumnDefinition; // NEW: Optional definition for TypeScript-first + generateMigration?: () => SQL; // NEW: Self-contained migration generation + }> +>; + +// Column knows how to generate its own SQL +export const generateColumnSQL = (column: ColumnSchemaComponent): string => { + if (!column.definition) return `${column.columnName} TEXT`; // fallback + + const { type, nullable, primaryKey, unique, length, defaultValue } = column.definition; + let sql = `${column.columnName} ${type.toUpperCase()}`; + + if (length) sql += `(${length})`; + if (primaryKey) sql += ' PRIMARY KEY'; + if (unique) sql += ' UNIQUE'; + if (!nullable) sql += ' NOT NULL'; + if (defaultValue !== undefined) sql += ` DEFAULT ${defaultValue}`; + + return sql; +}; +``` + +### Phase 2: Nested Schema Snapshot System + +**Complete directory structure**: +``` +.dumbo/ + snapshots/ # Current state + migration tracking + myapp.snapshot.ts # Root database snapshot + schemas/ + public.snapshot.ts # Schema-level snapshot + analytics.snapshot.ts + public/ + users.snapshot.ts # Table-level snapshots + posts.snapshot.ts + comments.snapshot.ts + analytics/ + events.snapshot.ts + metrics.snapshot.ts + migrations/ # All SQL migrations (generated or provided) + 2024-01-15-001-create_schema_public.sql + 2024-01-15-002-create_table_users.sql + 2024-01-15-003-create_index_idx_users_email.sql + 2024-01-16-001-add_email_to_users.sql +``` + +Note: Applied migrations are tracked in the database `migrations` table (see `migrator.ts` lines 167-189), not in files. + +**Migration SQL file examples**: + +```sql +-- .dumbo/migrations/2024-01-15-002-create_table_users.sql +-- Source: generated (from TypeScript definitions) +-- Component: sc:dumbo:table:users +-- Generated at: 2024-01-15T10:30:00Z +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) NOT NULL +); + +-- .dumbo/migrations/2024-01-16-001-custom_users_setup.sql +-- Source: provided (from component migrations array) +-- Component: sc:dumbo:table:users +-- Migration name: 001_custom_setup +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255), + internal_field JSONB +); +``` + +**Snapshot file examples with migration tracking**: + +```typescript +// SCENARIO 1: TypeScript-first (no migrations provided) +// Component: tableSchemaComponent({ +// tableName: 'users', +// columns: { id: column('id', { type: 'serial', primaryKey: true }) } +// }) + +// .dumbo/snapshots/schemas/public/users.snapshot.ts +export const usersSnapshot = { + tableName: 'users', + columns: { + id: { type: 'serial' as const, primaryKey: true }, + email: { type: 'varchar' as const, length: 255, nullable: false } + }, + indexes: { + idx_users_email: { columns: ['email'], unique: false } + }, + migrations: [] // No migrations provided, will be generated +} as const; + +// SCENARIO 2: SQL-first (only migrations provided) +// Component: tableSchemaComponent({ +// tableName: 'orders', +// migrations: [ +// sqlMigration('001_create', [SQL`CREATE TABLE orders ...`]), +// sqlMigration('002_add_user', [SQL`ALTER TABLE orders ...`]) +// ] +// }) + +// .dumbo/snapshots/schemas/public/orders.snapshot.ts +export const ordersSnapshot = { + tableName: 'orders', + columns: { + // Introspected from database after applying migrations + id: { type: 'serial' as const, primaryKey: false }, + total: { type: 'decimal' as const, precision: 10, scale: 2 }, + user_id: { type: 'int' as const, nullable: true } + }, + indexes: {}, + migrations: [ + { name: '001_create', sqls: ['CREATE TABLE orders (id SERIAL, total DECIMAL(10,2))'] }, + { name: '002_add_user', sqls: ['ALTER TABLE orders ADD COLUMN user_id INT'] } + ] +} as const; + +// SCENARIO 3: Hybrid (TypeScript definitions + SQL migrations) +// Component: tableSchemaComponent({ +// tableName: 'posts', +// columns: { id: column('id', ...), title: column('title', ...) }, +// migrations: [sqlMigration('001_custom', [SQL`CREATE TABLE posts ...`])] +// }) + +// .dumbo/snapshots/schemas/public/posts.snapshot.ts +export const postsSnapshot = { + tableName: 'posts', + columns: { + // TypeScript definitions as source of truth + id: { type: 'serial' as const, primaryKey: true }, + title: { type: 'varchar' as const, length: 255, nullable: false } + }, + indexes: {}, + migrations: [ + { name: '001_custom', sqls: ['CREATE TABLE posts (id SERIAL PRIMARY KEY, title VARCHAR(255), internal_field JSONB)'] } + ] +} as const; + +// .dumbo/snapshots/schemas/public.snapshot.ts +import { usersSnapshot } from './public/users.snapshot'; +import { postsSnapshot } from './public/posts.snapshot'; +import { commentsSnapshot } from './public/comments.snapshot'; + +export const publicSnapshot = { + schemaName: 'public', + tables: { + users: usersSnapshot, + posts: postsSnapshot, + comments: commentsSnapshot + } +} as const; + +// .dumbo/snapshots/myapp.snapshot.ts +import { publicSnapshot } from './schemas/public.snapshot'; +import { analyticsSnapshot } from './schemas/analytics.snapshot'; + +export const databaseSnapshot = { + databaseName: 'myapp', + version: '2024-01-15-001', + timestamp: '2024-01-15T10:30:00Z', + schemas: { + public: publicSnapshot, + analytics: analyticsSnapshot + } +} as const; +``` + +**Benefits of nested structure**: +- Minimizes git merge conflicts (each table in separate file) +- Clear ownership and change tracking +- Natural composition mirroring schema hierarchy +- Type-safe imports with TypeScript checking + +**New file to create**: `src/packages/dumbo/src/core/schema/snapshot/schemaSnapshot.ts` + +```typescript +export type ColumnSnapshot = { + type: ColumnDefinition['type']; + nullable?: boolean; + primaryKey?: boolean; + unique?: boolean; + length?: number; + precision?: number; + scale?: number; + defaultValue?: unknown; + references?: { + table: string; + column: string; + onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; + onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; + }; +}; + +export type IndexSnapshot = { + columns: string[]; + unique: boolean; + where?: string; +}; + +export type TableSnapshot = { + tableName: string; + columns: Record; + indexes: Record; + migrations: SQLMigration[]; // Track migrations defined in component +}; + +export type SchemaSnapshot = { + schemaName: string; + tables: Record; + migrations: SQLMigration[]; // Schema-level migrations +}; + +export type DatabaseSnapshot = { + databaseName: string; + version: string; + timestamp: string; + schemas: Record; + migrations: SQLMigration[]; // Database-level migrations +}; +``` + +### Phase 3: Component-Specific Migration Generation + +**IMPORTANT**: Each component is responsible for generating its own migrations when none are provided. + +#### Table Component Migration Generation + +**File to modify**: [src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts) + +```typescript +// Add to tableSchemaComponent.ts +export const generateTableMigrations = ( + table: TableSchemaComponent, + snapshot?: TableSnapshot +): SQLMigration[] => { + // If migrations already provided, return them (SQL-first mode) + if (table.migrations.length > 0) { + return table.migrations; + } + + // If no columns defined in TypeScript, nothing to generate + if (table.columns.size === 0) { + return []; + } + + const migrations: SQLMigration[] = []; + + if (!snapshot) { + // No snapshot = first time creation + const columnDefinitions = Array.from(table.columns.values()) + .map(col => generateColumnSQL(col)) + .join(',\n '); + + migrations.push( + sqlMigration(`create_table_${table.tableName}`, [ + SQL`CREATE TABLE IF NOT EXISTS ${table.tableName} ( + ${columnDefinitions} +)` + ]) + ); + } else { + // Generate ALTER statements based on diff with snapshot + const changes = diffTableWithSnapshot(table, snapshot); + + for (const newColumn of changes.addedColumns) { + migrations.push( + sqlMigration(`add_${newColumn.columnName}_to_${table.tableName}`, [ + SQL`ALTER TABLE ${table.tableName} ADD COLUMN IF NOT EXISTS ${generateColumnSQL(newColumn)}` + ]) + ); + } + } + + // Indexes are generated as separate migrations + for (const [name, index] of table.indexes) { + if (!snapshot || !snapshot.indexes[name]) { + migrations.push(generateIndexMigration(table.tableName, index)); + } + } + + return migrations; +}; + +// Table component enhanced constructor +export const tableSchemaComponent = (options: TableOptions): TableSchemaComponent => { + // ... existing implementation ... + + const component = { + // ... existing properties ... + generateMigrations: (snapshot?: TableSnapshot) => + generateTableMigrations(component, snapshot) + }; + + return component; +}; +``` + +#### Index Component Migration Generation + +**File to modify**: [src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts) + +```typescript +export const generateIndexMigration = ( + tableName: string, + index: IndexSchemaComponent +): SQLMigration => { + const uniqueClause = index.unique ? 'UNIQUE ' : ''; + const columns = index.columns.join(', '); + + return sqlMigration(`create_index_${index.indexName}`, [ + SQL`CREATE ${uniqueClause}INDEX IF NOT EXISTS ${index.indexName} + ON ${tableName}(${columns})` + ]); +}; +``` + +#### Database Schema Component Migration Generation + +**File to modify**: [src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts) + +```typescript +export const generateSchemaMigrations = ( + schema: DatabaseSchemaSchemaComponent, + snapshot?: SchemaSnapshot +): SQLMigration[] => { + if (schema.migrations.length > 0) { + return schema.migrations; + } + + const migrations: SQLMigration[] = []; + + // Create schema if doesn't exist + migrations.push( + sqlMigration(`create_schema_${schema.schemaName}`, [ + SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}` + ]) + ); + + // Tables will generate their own migrations + // But we collect them here for ordering + for (const table of schema.tables.values()) { + migrations.push(...table.generateMigrations(snapshot?.tables[table.tableName])); + } + + return migrations; +}; +``` + +### Phase 4: Tree Traversal Migration Collection + +**File to modify**: [src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts) + +The database component is the root and orchestrates migration generation by traversing the tree in the correct order: + +```typescript +export const collectAllMigrations = ( + database: DatabaseSchemaComponent, + snapshot?: DatabaseSnapshot +): SQLMigration[] => { + const migrations: SQLMigration[] = []; + + // Order matters! Follow this sequence: + // 1. Database-level migrations + // 2. Schema creation + // 3. Table creation (without foreign keys) + // 4. Column additions/modifications + // 5. Indexes + // 6. Foreign keys (future) + // 7. Other constraints (future) + + // If database has explicit migrations, use them + if (database.migrations.length > 0) { + return database.migrations; // SQL-first mode + } + + // 1. Database-level setup + migrations.push( + sqlMigration(`setup_database_${database.databaseName}`, [ + SQL`-- Database setup for ${database.databaseName}` + ]) + ); + + // 2. Traverse schemas + for (const schema of database.schemas.values()) { + const schemaSnapshot = snapshot?.schemas[schema.schemaName]; + + // Create schema + migrations.push( + sqlMigration(`create_schema_${schema.schemaName}`, [ + SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}` + ]) + ); + + // 3. Collect all tables first (structure only, no foreign keys) + const tableCreations: SQLMigration[] = []; + const indexCreations: SQLMigration[] = []; + + for (const table of schema.tables.values()) { + const tableSnapshot = schemaSnapshot?.tables[table.tableName]; + + if (table.migrations.length > 0) { + // Table has explicit migrations, use them + tableCreations.push(...table.migrations); + } else if (table.columns.size > 0) { + // Generate from TypeScript definitions + const tableMigrations = table.generateMigrations(tableSnapshot); + + // Separate table creation from index creation + tableMigrations.forEach(m => { + if (m.name.includes('index')) { + indexCreations.push(m); + } else { + tableCreations.push(m); + } + }); + } + } + + // Add in correct order + migrations.push(...tableCreations); // All tables first + migrations.push(...indexCreations); // Then all indexes + } + + // Future: Foreign key constraints would go here + // Future: Other constraints, triggers, etc. + + return migrations; +}; + +// Enhanced database component +export const databaseSchemaComponent = (options: DatabaseOptions): DatabaseSchemaComponent => { + // ... existing implementation ... + + const component = { + // ... existing properties ... + collectAllMigrations: (snapshot?: DatabaseSnapshot) => + collectAllMigrations(component, snapshot) + }; + + return component; +}; +``` + +### Phase 5: Schema Differ + +**New file to create**: `src/packages/dumbo/src/core/schema/differ/schemaDiffer.ts` + +```typescript +export type TableChanges = { + addedColumns: ColumnSchemaComponent[]; + removedColumns: string[]; + modifiedColumns: Array<{ + name: string; + from: ColumnSnapshot; + to: ColumnDefinition; + }>; +}; + +export const diffTableWithSnapshot = ( + table: TableSchemaComponent, + snapshot: TableSnapshot +): TableChanges => { + const changes: TableChanges = { + addedColumns: [], + removedColumns: [], + modifiedColumns: [] + }; + + // Find added columns + for (const [name, column] of table.columns) { + if (!snapshot.columns[name]) { + changes.addedColumns.push(column); + } + } + + // Find removed columns (might want to warn instead of remove) + for (const name in snapshot.columns) { + if (!table.columns.has(name)) { + changes.removedColumns.push(name); + } + } + + // Find modified columns + for (const [name, column] of table.columns) { + const snapshotColumn = snapshot.columns[name]; + if (snapshotColumn && !columnsEqual(column.definition, snapshotColumn)) { + changes.modifiedColumns.push({ + name, + from: snapshotColumn, + to: column.definition + }); + } + } + + return changes; +}; + +const columnsEqual = (def: ColumnDefinition | undefined, snap: ColumnSnapshot): boolean => { + if (!def) return false; + return ( + def.type === snap.type && + def.nullable === snap.nullable && + def.primaryKey === snap.primaryKey && + def.unique === snap.unique && + def.length === snap.length + ); +}; +``` + +### Phase 6: Type Generation from Schema + +**New file to create**: `src/packages/dumbo/src/core/schema/generators/typeGenerator.ts` + +```typescript +export const generateTypesFromSnapshot = (snapshot: DatabaseSnapshot): string => { + const lines: string[] = []; + + lines.push('// Auto-generated database types from schema snapshot'); + lines.push('// Do not edit manually - use dumbo schema:generate-types'); + lines.push(''); + + lines.push('export type DatabaseSchema = {'); + + for (const [schemaName, schema] of Object.entries(snapshot.schemas)) { + lines.push(` ${schemaName}: {`); + + for (const [tableName, table] of Object.entries(schema.tables)) { + lines.push(` ${tableName}: {`); + + for (const [columnName, column] of Object.entries(table.columns)) { + const tsType = sqlTypeToTypeScript(column); + lines.push(` ${columnName}: ${tsType};`); + } + + lines.push(' };'); + } + + lines.push(' };'); + } + + lines.push('};'); + + return lines.join('\n'); +}; + +const sqlTypeToTypeScript = (column: ColumnSnapshot): string => { + let baseType: string; + + switch (column.type) { + case 'serial': + case 'int': + case 'bigint': + baseType = 'number'; + break; + case 'varchar': + case 'text': + case 'uuid': + baseType = 'string'; + break; + case 'boolean': + baseType = 'boolean'; + break; + case 'timestamp': + baseType = 'Date'; + break; + case 'json': + case 'jsonb': + baseType = 'unknown'; + break; + default: + baseType = 'unknown'; + } + + return column.nullable ? `${baseType} | null` : baseType; +}; + +// Example generated output: +// export type DatabaseSchema = { +// public: { +// users: { +// id: number; +// email: string; +// name: string | null; +// }; +// posts: { +// id: number; +// userId: number; +// title: string; +// content: string | null; +// }; +// }; +// }; +``` + +### Phase 7: Migration Immutability Validation + +**New file to create**: `src/packages/dumbo/src/core/schema/validators/migrationImmutabilityValidator.ts` + +```typescript +import { SQLMigration } from '../sqlMigration'; + +export type ImmutabilityValidationResult = { + valid: boolean; + error?: string; +}; + +export const validateMigrationImmutability = ( + currentMigrations: ReadonlyArray, + snapshotMigrations: ReadonlyArray +): ImmutabilityValidationResult => { + // Check that no migrations were removed + for (const snapshotMigration of snapshotMigrations) { + const currentMigration = currentMigrations.find(m => m.name === snapshotMigration.name); + + if (!currentMigration) { + return { + valid: false, + error: `Migration '${snapshotMigration.name}' was removed from component!\n` + + `Migrations cannot be deleted once defined.\n` + + `If you need to undo a migration, add a new migration that reverses it.` + }; + } + + // Check that migration content hasn't changed + const snapshotSqls = snapshotMigration.sqls.map(sql => sql.toString()).join('\n'); + const currentSqls = currentMigration.sqls.map(sql => sql.toString()).join('\n'); + + if (snapshotSqls !== currentSqls) { + return { + valid: false, + error: `Migration '${snapshotMigration.name}' has been modified!\n` + + `Original SQL:\n${snapshotSqls}\n\n` + + `Current SQL:\n${currentSqls}\n\n` + + `Migrations must be immutable once defined. Add a new migration instead.` + }; + } + } + + return { valid: true }; +}; + +// Helper to detect new migrations +export const getNewMigrations = ( + currentMigrations: ReadonlyArray, + snapshotMigrations: ReadonlyArray +): SQLMigration[] => { + const snapshotNames = new Set(snapshotMigrations.map(m => m.name)); + return currentMigrations.filter(m => !snapshotNames.has(m.name)); +}; +``` + +### Phase 8: Schema Validation System + +**New file to create**: `src/packages/dumbo/src/core/schema/validators/migrationValidator.ts` + +```typescript +export type ValidationResult = { + valid: boolean; + errors: Array; + warnings: Array; +}; + +export type ValidationError = { + type: 'missing_column' | 'type_mismatch' | 'constraint_mismatch' | 'missing_table'; + message: string; + location: { + component: string; + file?: string; + line?: number; + }; + expected: string; + actual: string; +}; + +export type ValidationWarning = { + type: 'extra_column' | 'extra_index' | 'deprecated_type'; + message: string; +}; + +export const validateMigrationsAgainstSchema = ( + component: SchemaComponent, + options?: { strict: boolean } +): ValidationResult => { + const result: ValidationResult = { + valid: true, + errors: [], + warnings: [] + }; + + // Only validate if component has both TypeScript definitions AND migrations + if (!hasTypeScriptDefinitions(component) || component.migrations.length === 0) { + return result; + } + + // Simulate migrations in memory to get resulting schema + const resultingSchema = simulateMigrations(component.migrations); + + // Compare with TypeScript definitions + const comparison = compareSchemas(component, resultingSchema, options); + + // Generate detailed error messages + for (const mismatch of comparison.mismatches) { + result.errors.push({ + type: mismatch.type, + message: formatErrorMessage(mismatch), + location: { + component: component.schemaComponentKey, + file: mismatch.file, + line: mismatch.line + }, + expected: mismatch.expected, + actual: mismatch.actual + }); + result.valid = false; + } + + return result; +}; + +const formatErrorMessage = (mismatch: SchemaMismatch): string => { + return `Column '${mismatch.column}' mismatch in table '${mismatch.table}': + Expected: ${mismatch.expected} (from ${mismatch.tsFile}:${mismatch.tsLine}) + Actual: ${mismatch.actual} (from ${mismatch.sqlFile}:${mismatch.sqlLine}) + Fix: Either update TypeScript definition or add migration to match`; +}; +``` + +### Phase 9: CLI Commands + +**New CLI commands to implement**: + +```bash +# Compare current TypeScript definitions with snapshot +dumbo schema:diff + +# Generate migrations from diff +dumbo schema:generate [name] +# Example: dumbo schema:generate add-email-to-users + +# Update snapshot to current state +dumbo schema:snapshot + +# Generate TypeScript types from snapshot +dumbo schema:generate-types + +# Validate migrations against TypeScript definitions +dumbo schema:validate + +# Show current schema state +dumbo schema:status + +# Collect and run all migrations from database component +dumbo schema:migrate +``` + +### Phase 10: Migration Orchestrator + +**New file to create**: `src/packages/dumbo/src/core/schema/orchestrator/migrationOrchestrator.ts` + +```typescript +export class MigrationOrchestrator { + constructor( + private database: DatabaseSchemaComponent, + private snapshotPath: string = '.dumbo/snapshots', + private migrationsPath: string = '.dumbo/migrations' + ) {} + + async processMigrations(): Promise { + // Load latest snapshot if exists + const snapshot = await this.loadSnapshot(); + + // Validate migration immutability + const validation = await this.validateMigrations(snapshot); + if (!validation.valid) { + throw new Error(validation.error); + } + + // Process user-provided migrations + const userProvidedMigrations = await this.processUserProvidedMigrations(snapshot); + + // Generate migrations for TypeScript-defined components + const generatedMigrations = await this.generateMigrationsFromDefinitions(snapshot); + + return { + userProvided: userProvidedMigrations, + generated: generatedMigrations, + all: [...userProvidedMigrations, ...generatedMigrations] + }; + } + + private async processUserProvidedMigrations(snapshot: DatabaseSnapshot | null): Promise { + const newMigrations: SQLMigration[] = []; + + // Process database-level migrations + if (this.database.migrations.length > 0) { + const snapshotMigrations = snapshot?.migrations || []; + const newDbMigrations = getNewMigrations(this.database.migrations, snapshotMigrations); + + for (const migration of newDbMigrations) { + // Write to migrations folder + await this.writeMigrationToFile(migration, 'database', this.database.schemaComponentKey); + newMigrations.push(migration); + } + } + + // Traverse and process schema and table migrations + for (const schema of this.database.schemas.values()) { + const schemaSnapshot = snapshot?.schemas[schema.schemaName]; + + // Process schema migrations + if (schema.migrations.length > 0) { + const snapshotMigrations = schemaSnapshot?.migrations || []; + const newSchemaMigrations = getNewMigrations(schema.migrations, snapshotMigrations); + + for (const migration of newSchemaMigrations) { + await this.writeMigrationToFile(migration, 'schema', schema.schemaComponentKey); + newMigrations.push(migration); + } + } + + // Process table migrations + for (const table of schema.tables.values()) { + const tableSnapshot = schemaSnapshot?.tables[table.tableName]; + + if (table.migrations.length > 0) { + const snapshotMigrations = tableSnapshot?.migrations || []; + const newTableMigrations = getNewMigrations(table.migrations, snapshotMigrations); + + for (const migration of newTableMigrations) { + await this.writeMigrationToFile(migration, 'table', table.schemaComponentKey); + newMigrations.push(migration); + } + } + } + } + + return newMigrations; + } + + private async writeMigrationToFile( + migration: SQLMigration, + type: 'database' | 'schema' | 'table', + componentKey: string + ): Promise { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `${timestamp}-${migration.name}.sql`; + + const header = [ + `-- Source: provided (from component migrations array)`, + `-- Component: ${componentKey}`, + `-- Type: ${type}`, + `-- Migration name: ${migration.name}`, + `-- Written at: ${new Date().toISOString()}`, + '', + '' + ].join('\n'); + + const content = header + migration.sqls.map(sql => sql.toString()).join(';\n') + ';'; + + await writeFile(`${this.migrationsPath}/${filename}`, content); + } + + private async validateMigrations(snapshot: DatabaseSnapshot | null): Promise { + if (!snapshot) return { valid: true }; + + // Validate database migrations + const dbValidation = validateMigrationImmutability( + this.database.migrations, + snapshot.migrations || [] + ); + if (!dbValidation.valid) return dbValidation; + + // Validate schema and table migrations + for (const schema of this.database.schemas.values()) { + const schemaSnapshot = snapshot.schemas[schema.schemaName]; + if (!schemaSnapshot) continue; + + const schemaValidation = validateMigrationImmutability( + schema.migrations, + schemaSnapshot.migrations || [] + ); + if (!schemaValidation.valid) return schemaValidation; + + for (const table of schema.tables.values()) { + const tableSnapshot = schemaSnapshot.tables[table.tableName]; + if (!tableSnapshot) continue; + + const tableValidation = validateMigrationImmutability( + table.migrations, + tableSnapshot.migrations || [] + ); + if (!tableValidation.valid) return tableValidation; + } + } + + return { valid: true }; + } + + async runMigrations(dumbo: Dumbo): Promise { + const processed = await this.processMigrations(); + + // Group migrations by type for correct ordering + const grouped = this.groupMigrations(processed.all); + + // Run in correct order + await this.runMigrationGroup(dumbo, grouped.schemas); + await this.runMigrationGroup(dumbo, grouped.tables); + await this.runMigrationGroup(dumbo, grouped.columns); + await this.runMigrationGroup(dumbo, grouped.indexes); + // Future: grouped.foreignKeys, grouped.constraints + + // Update snapshot after successful migration + await this.updateSnapshot(); + } + + private async loadSnapshot(): Promise { + try { + // Load root snapshot which imports all nested snapshots + const module = await import(`${this.snapshotPath}/${this.database.databaseName}.snapshot.ts`); + return module.databaseSnapshot; + } catch (error) { + // No snapshot found, will generate everything from scratch + return null; + } + } + + private async updateSnapshot(): Promise { + // Generate new snapshots for each component + await this.generateDatabaseSnapshot(this.database); + } + + private async generateDatabaseSnapshot(database: DatabaseSchemaComponent): Promise { + // Generate nested snapshot files + for (const schema of database.schemas.values()) { + await this.generateSchemaSnapshot(schema); + } + + // Generate root database snapshot that imports schemas + const imports = Array.from(database.schemas.values()) + .map(s => `import { ${s.schemaName}Snapshot } from './schemas/${s.schemaName}.snapshot';`) + .join('\n'); + + const migrationsStr = this.formatMigrationsForSnapshot(database.migrations); + + const content = `${imports} + +export const databaseSnapshot = { + databaseName: '${database.databaseName}', + version: '${new Date().toISOString().split('T')[0]}-001', + timestamp: '${new Date().toISOString()}', + schemas: { + ${Array.from(database.schemas.values()) + .map(s => `${s.schemaName}: ${s.schemaName}Snapshot`) + .join(',\n ')} + }, + migrations: ${migrationsStr} +} as const;`; + + await writeFile(`${this.snapshotPath}/${database.databaseName}.snapshot.ts`, content); + } + + private async generateSchemaSnapshot(schema: DatabaseSchemaSchemaComponent): Promise { + // Generate table snapshots first + for (const table of schema.tables.values()) { + await this.generateTableSnapshot(schema.schemaName, table); + } + + // Generate schema snapshot that imports tables + const imports = Array.from(schema.tables.values()) + .map(t => `import { ${t.tableName}Snapshot } from './${schema.schemaName}/${t.tableName}.snapshot';`) + .join('\n'); + + const migrationsStr = this.formatMigrationsForSnapshot(schema.migrations); + + const content = `${imports} + +export const ${schema.schemaName}Snapshot = { + schemaName: '${schema.schemaName}', + tables: { + ${Array.from(schema.tables.values()) + .map(t => `${t.tableName}: ${t.tableName}Snapshot`) + .join(',\n ')} + }, + migrations: ${migrationsStr} +} as const;`; + + await writeFile(`${this.snapshotPath}/schemas/${schema.schemaName}.snapshot.ts`, content); + } + + private async generateTableSnapshot(schemaName: string, table: TableSchemaComponent): Promise { + const migrationsStr = this.formatMigrationsForSnapshot(table.migrations); + + const content = `export const ${table.tableName}Snapshot = { + tableName: '${table.tableName}', + columns: { + ${Array.from(table.columns.entries()) + .map(([name, col]) => this.generateColumnSnapshot(name, col)) + .join(',\n ')} + }, + indexes: { + ${Array.from(table.indexes.entries()) + .map(([name, idx]) => this.generateIndexSnapshot(name, idx)) + .join(',\n ')} + }, + migrations: ${migrationsStr} +} as const;`; + + await writeFile(`${this.snapshotPath}/schemas/${schemaName}/${table.tableName}.snapshot.ts`, content); + } + + private formatMigrationsForSnapshot(migrations: ReadonlyArray): string { + if (migrations.length === 0) return '[]'; + + const migrationStrs = migrations.map(m => { + const sqlsStr = m.sqls.map(sql => `'${sql.toString().replace(/'/g, "\\'")}'`).join(', '); + return `{ name: '${m.name}', sqls: [${sqlsStr}] }`; + }); + + return `[\n ${migrationStrs.join(',\n ')}\n ]`; + } + + private groupMigrations(migrations: SQLMigration[]) { + return { + schemas: migrations.filter(m => m.name.includes('schema')), + tables: migrations.filter(m => m.name.includes('create_table')), + columns: migrations.filter(m => m.name.includes('add_') || m.name.includes('alter_')), + indexes: migrations.filter(m => m.name.includes('index')), + }; + } +} +``` + +## Testing Strategy + +### Unit Tests + +Following the pattern in [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts): + +```typescript +// src/packages/dumbo/src/core/schema/generators/migrationGenerator.unit.spec.ts +import { describe, it, assert } from 'vitest'; +import { tableSchemaComponent } from '../components/tableSchemaComponent'; +import { column } from '../components/columnSchemaComponent'; + +describe('generateTableMigrations', () => { + it('should generate CREATE TABLE from TypeScript definition', () => { + const table = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255, nullable: false }) + } + }); + + const migrations = table.generateMigrations(); + + assert.equal(migrations.length, 1); + assert.match(migrations[0].sqls[0], /CREATE TABLE IF NOT EXISTS users/); + assert.match(migrations[0].sqls[0], /id SERIAL PRIMARY KEY/); + assert.match(migrations[0].sqls[0], /email VARCHAR\(255\) NOT NULL/); + }); + + it('should detect added columns when comparing with snapshot', () => { + const snapshot = { + tableName: 'users', + columns: { + id: { type: 'serial' as const, primaryKey: true } + }, + indexes: {} + }; + + const table = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255 }) + } + }); + + const migrations = table.generateMigrations(snapshot); + + assert.equal(migrations.length, 1); + assert.match(migrations[0].sqls[0], /ALTER TABLE users ADD COLUMN IF NOT EXISTS email/); + }); +}); +``` + +### Integration Tests + +```typescript +// src/packages/dumbo/src/core/schema/orchestrator/migrationOrchestrator.integration.spec.ts +import { describe, it, assert, beforeEach } from 'vitest'; +import { MigrationOrchestrator } from './migrationOrchestrator'; +import { createTestDatabase } from '../../test/utils'; + +describe('MigrationOrchestrator integration', () => { + let db: Dumbo; + + beforeEach(async () => { + db = await createTestDatabase(); + }); + + it('should apply schema changes in correct order', async () => { + const schema = databaseSchemaComponent({ + databaseName: 'test', + schemas: { + public: databaseSchemaSchemaComponent({ + schemaName: 'public', + tables: { + users: tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255 }) + } + }), + posts: tableSchemaComponent({ + tableName: 'posts', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + userId: column('userId', { type: 'int', references: { table: 'users', column: 'id' } }) + } + }) + } + }) + } + }); + + const orchestrator = new MigrationOrchestrator(schema); + await orchestrator.runMigrations(db); + + // Verify tables were created + const tables = await db.query(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + `); + + assert.deepEqual(tables.map(t => t.table_name).sort(), ['posts', 'users']); + + // Verify columns exist + const columns = await db.query(` + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = 'users' + `); + + assert.equal(columns.length, 2); + }); +}); +``` + +### End-to-End Tests + +```typescript +// src/packages/dumbo/src/core/schema/e2e/schemaEvolution.e2e.spec.ts +describe('Schema evolution E2E', () => { + it('should handle complete schema lifecycle', async () => { + // 1. Initial schema definition + const v1Schema = createSchema({ version: 1 }); + await orchestrator.runMigrations(v1Schema); + await orchestrator.updateSnapshot(); + + // 2. Schema modification + const v2Schema = createSchema({ + version: 2, + addColumn: { table: 'users', column: 'phone' } + }); + + // 3. Diff detection + const diff = await orchestrator.diff(v2Schema); + assert.equal(diff.changes.length, 1); + assert.equal(diff.changes[0].type, 'add_column'); + + // 4. Migration generation + const migrations = await orchestrator.generateMigrations(); + assert.equal(migrations.length, 1); + + // 5. Migration application + await orchestrator.runMigrations(v2Schema); + + // 6. Snapshot update + await orchestrator.updateSnapshot(); + + // 7. Type generation + const types = await generateTypesFromSnapshot(); + assert.match(types, /phone: string | null/); + + // 8. Validation + const validation = await validateMigrationsAgainstSchema(v2Schema); + assert.equal(validation.valid, true); + }); +}); +``` + +## Complete Flow Example + +This section demonstrates how all pieces work together in practice: + +### Step 1: Initial Component Definition with SQL Migration + +```typescript +// User defines component with custom SQL migration +const usersTable = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255 }) + }, + migrations: [ + sqlMigration('001_custom_create', [ + SQL`CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + internal_tracking JSONB DEFAULT '{}' + )` + ]) + ] +}); +``` + +### Step 2: First Orchestrator Run + +```typescript +const orchestrator = new MigrationOrchestrator(database); + +// Process migrations (validates, diffs, writes new ones) +await orchestrator.processMigrations(); +// - No snapshot exists, so no validation needed +// - Detects '001_custom_create' as new +// - Writes to: .dumbo/migrations/2024-01-15-001-custom_create.sql +``` + +### Step 3: Generated Files After First Run + +```sql +-- .dumbo/migrations/2024-01-15-001-custom_create.sql +-- Source: provided (from component migrations array) +-- Component: sc:dumbo:table:users +-- Type: table +-- Migration name: 001_custom_create +-- Written at: 2024-01-15T10:30:00Z + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + internal_tracking JSONB DEFAULT '{}' +); +``` + +```typescript +// .dumbo/snapshots/schemas/public/users.snapshot.ts +export const usersSnapshot = { + tableName: 'users', + columns: { + id: { type: 'serial' as const, primaryKey: true }, + email: { type: 'varchar' as const, length: 255, nullable: false } + }, + indexes: {}, + migrations: [ + { + name: '001_custom_create', + sqls: ['CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT \'{}\')'] + } + ] +} as const; +``` + +### Step 4: User Adds New Column and Migration + +```typescript +// User updates component +const usersTable = tableSchemaComponent({ + tableName: 'users', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + email: column('email', { type: 'varchar', length: 255 }), + name: column('name', { type: 'varchar', length: 100 }) // NEW! + }, + migrations: [ + sqlMigration('001_custom_create', [ + SQL`CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) UNIQUE, + internal_tracking JSONB DEFAULT '{}' + )` + ]), + sqlMigration('002_add_name', [ // NEW! + SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)` + ]) + ] +}); +``` + +### Step 5: Second Orchestrator Run + +```typescript +await orchestrator.processMigrations(); +// 1. Validates '001_custom_create' hasn't changed ✓ +// 2. Detects '002_add_name' as new +// 3. Writes new migration to file +// 4. NO generated migration for 'name' column (user provided SQL) +``` + +### Step 6: Error Case - Modified Migration + +```typescript +// User accidentally modifies existing migration (BAD!) +const usersTable = tableSchemaComponent({ + tableName: 'users', + migrations: [ + sqlMigration('001_custom_create', [ + SQL`CREATE TABLE users ( + id BIGSERIAL PRIMARY KEY, // Changed from SERIAL! + email VARCHAR(255) UNIQUE, + internal_tracking JSONB DEFAULT '{}' + )` + ]), + sqlMigration('002_add_name', [ + SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)` + ]) + ] +}); + +await orchestrator.processMigrations(); +// Throws error: +// Migration '001_custom_create' has been modified! +// Original SQL: +// CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}') +// +// Current SQL: +// CREATE TABLE users (id BIGSERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}') +// +// Migrations must be immutable once defined. Add a new migration instead. +``` + +### Step 7: Mixed Mode - TypeScript + SQL + +```typescript +// Another table with TypeScript-first approach +const postsTable = tableSchemaComponent({ + tableName: 'posts', + columns: { + id: column('id', { type: 'serial', primaryKey: true }), + title: column('title', { type: 'varchar', length: 255 }), + userId: column('userId', { type: 'int', references: { table: 'users', column: 'id' } }) + } + // No migrations provided - will be generated! +}); + +await orchestrator.processMigrations(); +// Generates and writes: .dumbo/migrations/2024-01-15-002-create_table_posts.sql +// Content: +// -- Source: generated (from TypeScript definitions) +// -- Component: sc:dumbo:table:posts +// -- Generated at: 2024-01-15T10:45:00Z +// CREATE TABLE IF NOT EXISTS posts ( +// id SERIAL PRIMARY KEY, +// title VARCHAR(255) NOT NULL, +// userId INT REFERENCES users(id) +// ); +``` + +### Step 8: Type Generation + +```typescript +// Generate types from snapshots +const types = await generateTypesFromSnapshot(databaseSnapshot); + +// Results in: +export type DatabaseSchema = { + public: { + users: { + id: number; + email: string; + name: string | null; + }; + posts: { + id: number; + title: string; + userId: number; + }; + }; +}; + +// Can be used in Pongo query builder: +const result = await pongo + .from('users') + .where('email', '=', 'test@example.com') + .select(['id', 'name']); +// TypeScript knows result is { id: number; name: string | null }[] +``` + +## Migration Strategy for Existing Code + +1. **Backward Compatible**: Existing SQL-first code continues to work unchanged +2. **Progressive Enhancement**: Can add TypeScript definitions gradually +3. **Opt-in Validation**: Validation only runs when explicitly enabled +4. **Safe Defaults**: Use `CREATE IF NOT EXISTS` when no snapshot found + +## Success Criteria + +1. ✅ Each component can generate its own migrations +2. ✅ Database component traverses tree in correct order +3. ✅ Can use raw SQL migrations without TypeScript (backward compatible) +4. ✅ Can validate SQL migrations against TypeScript definitions +5. ✅ Nested snapshot system minimizes merge conflicts +6. ✅ Types can be generated from schema for Pongo integration +7. ✅ CLI provides clear workflow for schema evolution +8. ✅ Type safety maintained throughout +9. ✅ No breaking changes to existing API + +## Key Architectural Decisions + +1. **Each component generates its own migrations** - Encapsulation and single responsibility +2. **Database component orchestrates collection** - Single entry point for migration generation +3. **Tree traversal follows dependency order** - Schemas → Tables → Columns → Indexes → Foreign Keys +4. **TypeScript definitions are source of truth when present** - Override SQL for intended state +5. **Nested snapshot structure** - Each table/schema in separate file to minimize conflicts +6. **TypeScript snapshot format** - Type safety and IDE support over JSON simplicity +7. **Snapshots track component migrations** - Enable detection of new user-provided migrations +8. **All migrations go to `.dumbo/migrations/`** - Both generated and user-provided, uniform handling +9. **Migration immutability enforced** - Once in snapshot, migrations cannot be modified +10. **CREATE IF NOT EXISTS as default** - Safe behavior when no snapshot exists +11. **Indexes generate as separate migrations** - Clean separation of concerns +12. **Validation is opt-in** - No breaking changes, teams choose their strictness level +13. **Type generation from snapshots** - Enable strongly-typed Pongo query builders + +## Implementation Order + +1. **Phase 1**: Column definitions with SQL generation +2. **Phase 2**: Nested snapshot system structure with migration tracking +3. **Phase 3**: Component-specific migration generation (each component file) +4. **Phase 4**: Tree traversal in database component +5. **Phase 5**: Schema differ for change detection +6. **Phase 6**: Type generation from schema snapshots +7. **Phase 7**: Migration immutability validation +8. **Phase 8**: Schema validation system with detailed errors +9. **Phase 9**: CLI commands for workflow +10. **Phase 10**: Migration orchestrator with diffing and validation + +Each phase builds on the previous one and can be tested independently. The key insight is that each component knows how to generate its own SQL, and the database component knows how to traverse the tree in the correct order to collect all migrations. + +## Future Enhancements (Out of Scope) + +1. **Query Builder Integration**: Use generated types for strongly-typed Pongo queries +2. **Schema Introspection**: Read current schema from database and generate TypeScript +3. **Migration Rollback**: Track down migrations and generate reversal scripts +4. **Multi-Database Support**: PostgreSQL, MySQL, SQLite-specific features +5. **Partial Schema Definitions**: Allow defining subset of columns for gradual adoption +6. **Smart Merge Tool**: CLI command to help resolve snapshot conflicts +7. **Performance Optimization**: Cache component trees for large schemas + +## Summary + +This plan unifies SQL migrations and TypeScript schema definitions through: + +### Core Innovations +1. **Dual-purpose snapshots** - Track both current schema state AND component migrations +2. **Migration diffing** - Detect new user-provided migrations by comparing with snapshots +3. **Immutability enforcement** - Prevent dangerous modification of existing migrations +4. **Unified migration storage** - All migrations (generated/provided) go to `.dumbo/migrations/` +5. **Nested snapshot structure** - Minimize git conflicts by separating tables into files + +### Key Benefits +- **Flexibility** - Support SQL-first, TypeScript-first, or hybrid approaches +- **Safety** - Migration immutability prevents accidental schema corruption +- **Type Safety** - Generate types from snapshots for Pongo query builders +- **Gradual Adoption** - Add TypeScript definitions progressively to SQL-first projects +- **Clear Audit Trail** - Track all migrations with source metadata + +### Migration Workflow +1. Component defines schema (TypeScript) and/or migrations (SQL) +2. Orchestrator validates immutability against snapshot +3. New user-provided migrations are written to files +4. TypeScript definitions generate missing migrations +5. Snapshots update to reflect current state + migrations +6. Types are generated for strongly-typed queries + +This architecture provides a robust foundation for schema evolution while maintaining backwards compatibility and enabling progressive enhancement. + +## References + +- Component hierarchy uses URN system with `sc:dumbo:database:{name}` pattern +- Migration bubbling happens at lines 79-83 in schemaComponent.ts +- Recent refactoring in commit 03fb40a changed to use `Exclude` instead of `Omit` +- Test patterns follow existing dumboSchema.unit.spec.ts structure +- Integration target is Pongo schema system in src/packages/pongo/src/core/schema/index.ts \ No newline at end of file diff --git a/src/packages/dumbo/src/core/schema/REVIEW.MD b/src/packages/dumbo/src/core/schema/REVIEW.MD new file mode 100644 index 00000000..771dda1b --- /dev/null +++ b/src/packages/dumbo/src/core/schema/REVIEW.MD @@ -0,0 +1,201 @@ +Deep Analysis of the Architecture +After thoroughly rereading the plan, here's my multi-perspective analysis: +From a Regular User's Perspective (Storage Library Consumer) +Strengths: +Progressive Disclosure of Complexity - Users can start simple (SQL-only or TypeScript-only) and gradually adopt more features. This is excellent API design. +Familiar Patterns - The API resembles popular ORMs (Prisma, TypeORM) making adoption easier. +Clear Error Messages - The immutability validation provides actionable feedback rather than cryptic errors. +Type Safety Benefits - The generated types for query builders provide immediate value that developers can feel in their IDE. +Potential Pain Points: +Hidden Complexity in Snapshots - The dual-purpose snapshots (schema state + migration tracking) might be confusing initially. Users might not understand why snapshots contain their migrations. +Directory Proliferation - The .dumbo/ folder with nested snapshots could feel overwhelming for small projects. +Mental Model Shift - Understanding when TypeScript is truth vs when SQL is truth requires careful documentation and good examples. +Migration Immutability - While safer, this might frustrate users coming from Rails/Django where you can edit migrations before deploying. +From a Software Architect's Perspective +Architectural Strengths: +Separation of Concerns - Each component generating its own SQL is textbook single responsibility principle. This makes the system highly testable and maintainable. +Command-Query Separation - The clear distinction between: +Commands (processMigrations, runMigrations) +Queries (loadSnapshot, validateMigrations) +Pure functions (generateMigrations, diffTableWithSnapshot) +Temporal Decoupling - Snapshots act as a temporal boundary, allowing the system to reason about "then vs now" without querying the database. +Composability - The tree structure with proper traversal order handles complex dependencies elegantly. +Architectural Concerns: +Snapshot as Dual Source of Truth - Using snapshots for BOTH schema state AND migration tracking violates single responsibility. Consider splitting into: +Schema snapshots (current state for diffing) +Migration manifest (tracking what's been processed) +File System as Database - The .dumbo/migrations/ directory becomes a critical data store. What happens with: +Concurrent writes? +File system permissions? +CI/CD environments? +Type Generation Timing - Generating types from snapshots rather than from the database means types might not reflect reality if migrations fail partially. +Validation vs Enforcement - The system validates but doesn't enforce schema consistency. A team member could still manually edit the database. +Architecture as Code Analysis +Elegant Design Patterns: +Builder Pattern in dumboSchema API: +dumboSchema.database('myapp', { +public: dumboSchema.schema('public', { +users: dumboSchema.table('users', {...}) +}) +}) +Template Method Pattern in migration generation - each component implements its own generateMigrations() but follows the same structure. +Visitor Pattern in tree traversal - the orchestrator visits each component in dependency order. +Memento Pattern - Snapshots act as mementos capturing component state at a point in time. +Code Smells to Address: +Large Parameter Lists - The orchestrator constructor and several functions have many parameters that could be grouped into configuration objects. +String Manipulation for SQL - Building SQL through string concatenation (even with template literals) is risky. Consider a SQL builder abstraction. +Implicit Ordering Dependencies - The migration grouping relies on naming conventions (includes('schema'), includes('create_table')). This is fragile. +Mixed Abstraction Levels - The orchestrator handles both high-level workflow and low-level file I/O. Consider extracting a MigrationWriter service. +Philosophical Considerations +The Snapshot Paradox: The snapshots serve as both historical record (what migrations were defined) and current state (what the schema looks like). This dual nature is philosophically interesting but practically complex. It's like Git storing both diffs AND the current state in the same structure. Migration Immutability Trade-off: Enforcing immutability is philosophically sound (history should be append-only) but practically challenging. Real-world schema evolution is messy, and sometimes you need to fix a broken migration. TypeScript as Truth: The decision to make TypeScript the source of truth when present is bold. It prioritizes developer experience over database reality, which could lead to drift. +Recommendations for Improvement +Split Snapshot Responsibilities: +Keep schema snapshots for state +Add a separate .dumbo/manifest.json for tracking processed migrations +Add Schema Verification: +After running migrations, query information_schema to verify the result matches expectations +Generate types from actual database state, not snapshots +Implement Conflict Resolution: +Add a dumbo schema:merge command for snapshot conflicts +Use three-way merge logic (base, mine, theirs) +Consider Event Sourcing: +Treat migrations as events in an event store +Snapshots become projections of those events +Enables time-travel debugging +Add Dry-Run Capability: +Let users preview what migrations would be generated/applied +Essential for production deployments +Overall Assessment +This is a sophisticated and well-thought-out architecture that successfully bridges the gap between SQL-first and TypeScript-first workflows. The use of snapshots for migration tracking is innovative, though it adds complexity. The system would benefit from: +Clearer separation between state tracking and migration manifest +Runtime verification that generated types match database reality +Better tooling for snapshot conflict resolution +More explicit handling of edge cases (partial migration failure, concurrent modifications) +The architecture shows maturity in its backwards compatibility and progressive enhancement approach. It's clearly designed by someone who understands both the technical challenges and the human factors of schema management. Grade: B+ Strong foundation with room for refinement in the snapshot dual-responsibility and runtime verification aspects. + +######### + +What if snapshots were stored as jsons would that help in understanding the split? +Excellent question! Let me think through how JSON snapshots would affect the architecture, particularly around the split of responsibilities. +JSON Snapshots: Impact Analysis +How JSON Would Help with the Split +Current Issue with TypeScript Snapshots: +// .dumbo/snapshots/schemas/public/users.snapshot.ts +export const usersSnapshot = { +tableName: 'users', +columns: { /_ schema state _/ }, +migrations: [ /* migration tracking */ ] // Mixed responsibility! +} as const; +With JSON + Split Responsibilities: +// .dumbo/snapshots/schemas/public/users.schema.json +{ +"tableName": "users", +"columns": { +"id": { "type": "serial", "primaryKey": true }, +"email": { "type": "varchar", "length": 255 } +}, +"indexes": {} +} + +// .dumbo/manifests/schemas/public/users.manifest.json +{ +"component": "sc:dumbo:table:users", +"processedMigrations": [ +{ +"name": "001_custom_create", +"hash": "sha256:abcd1234...", +"processedAt": "2024-01-15T10:30:00Z", +"source": "provided" +} +] +} +Benefits of JSON Format +Clear Separation of Concerns +_.schema.json = Current state (for diffing) +_.manifest.json = Migration history (for tracking) +No temptation to mix responsibilities in a single file +Better Git Diffs +{ +"columns": { +"id": { "type": "serial" }, + +- "email": { "type": "varchar", "length": 255 } + } + } + Much cleaner than TypeScript object diffs with imports and type annotations. + Language Agnostic + Other tools (Python scripts, CI/CD, database admin tools) can read/write + No TypeScript compilation needed to access snapshot data + Simpler Merge Resolution + JSON merge tools are mature and widespread + Can use jq or similar for command-line manipulation + Easier to write custom merge strategies + Drawbacks of JSON + Loss of Type Safety + No compile-time checking of snapshot structure + Need runtime validation (though you need this anyway) + No Imports/Composition + Can't use TypeScript's import system for nested structure + Would need to manually resolve file references + Verbosity for Complex Structures + No as const assertions + More verbose without TypeScript's type inference + Hybrid Approach: Best of Both Worlds + What if we used JSON for storage but TypeScript for types? + // .dumbo/snapshots/types.ts (generated) + import usersSchema from './schemas/public/users.schema.json'; + import postsSchema from './schemas/public/posts.schema.json'; + +export type DatabaseSchema = { +public: { +users: typeof usersSchema; +posts: typeof postsSchema; +} +}; +Recommended Structure with JSON +.dumbo/ +schemas/ # Current state (JSON) +database.json +public/ +schema.json +tables/ +users.json +posts.json +manifests/ # Migration tracking (JSON) +database.json +public/ +schema.json +tables/ +users.json +posts.json +migrations/ # SQL files +2024-01-15-001-create.sql +generated/ # Generated TypeScript +types.ts # Generated from schemas/ +manifest-types.ts # Types for manifest structure +Impact on the Architecture +Cleaner Responsibilities: +Schema JSON: Pure state representation +Manifest JSON: Pure migration history +Generated TS: Type safety without mixing concerns +Better Tooling Opportunities: +dumbo schema:diff - Compare JSON files +dumbo manifest:verify - Check manifest against database +dumbo schema:validate - JSON schema validation +Simpler Mental Model: +"Schemas describe what IS" +"Manifests describe what WAS DONE" +"Migrations describe HOW TO DO IT" +My Recommendation +Yes, JSON would significantly help with understanding and implementing the split. Specifically: +Use JSON for snapshots (both schema and manifest) +Keep them in separate directories +Generate TypeScript types from the JSON for type safety +Store the actual SQL in migrations as a third, distinct concern +This gives you: +Clear separation of concerns +Better git diffs and merge resolution +Language-agnostic data files +Type safety through code generation +Simpler mental model +The only real loss is the elegance of TypeScript imports for composition, but that's a worthwhile trade-off for the clarity gained. Would you like me to revise the plan to use this JSON-based approach with properly separated concerns? From d247020693bc8378ed5341277431571f52153a6a Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Fri, 31 Oct 2025 21:05:58 +0100 Subject: [PATCH 19/39] Added docs for design of feature schema component --- .../dumbo/src/core/schema/dumboFeature.md | 1606 +++++++++++++++++ .../dumbo/src/core/schema/schema_feature.md | 1197 ++++++++++++ 2 files changed, 2803 insertions(+) create mode 100644 src/packages/dumbo/src/core/schema/dumboFeature.md create mode 100644 src/packages/dumbo/src/core/schema/schema_feature.md diff --git a/src/packages/dumbo/src/core/schema/dumboFeature.md b/src/packages/dumbo/src/core/schema/dumboFeature.md new file mode 100644 index 00000000..eaccab5b --- /dev/null +++ b/src/packages/dumbo/src/core/schema/dumboFeature.md @@ -0,0 +1,1606 @@ +read src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md first, then tell me how would you recommend to join it with src/packages/pongo/src/core/schema/index.ts where pongo database could be either database or database per database schema. Also how would that fit Architecture as Code defined as: + +export type EmmettRelationshipType = string; + +export type PortRequirements = Record; + +export type EmmettArchModule< +Requires extends PortRequirements | undefined = undefined, +Exposes extends PortRequirements | undefined = undefined, + +> = { +> name: string; +> } & (Requires extends undefined +> ? Exposes extends undefined + + ? { ports?: undefined } // both undefined: ports optional + : { ports: { exposes: Exposes } } // only Exposes defined + +: Exposes extends undefined +? { ports: { requires: Requires } } // only Requires defined +: { ports: { requires: Requires; exposes: Exposes } }); // both defined + +export type AnyEmmettArchModule = EmmettArchModule; + +export interface EmmettRelationship< +Source extends AnyEmmettArchModule = AnyEmmettArchModule, +Target extends AnyEmmettArchModule = AnyEmmettArchModule, + +> { +> source: Source['name']; +> target: Target['name']; +> type: EmmettRelationshipType; +> description?: string; +> } + +export type EmmettRelationshipsMap< +Source extends AnyEmmettArchModule = AnyEmmettArchModule, +Target extends AnyEmmettArchModule = AnyEmmettArchModule, + +> = Record>; + +export type EmmettComponent< +NestedComponents extends +// eslint-disable-next-line @typescript-eslint/no-explicit-any +Record> | undefined = undefined, +Requires extends PortRequirements | undefined = undefined, +Exposes extends PortRequirements | undefined = undefined, + +> = EmmettArchModule & +> (NestedComponents extends undefined + + ? { components?: undefined } + : { components: NestedComponents }); + +export interface EmmettContainer< +T extends Record = Record, + +> extends EmmettArchModule { +> components?: T; +> } + +export interface EmmettSystem< +T extends Record = Record, + +> extends EmmettArchModule { +> containers?: T; +> } + +// export type EmmettComponentsMap> = { +// [K in keyof T]: EmmettComponent< +// T[K] extends EmmettComponent ? U : unknown +// >; +// }; + +export type EmmettContainersMap> = { +[K in keyof T]: EmmettContainer< +T[K] extends EmmettContainer ? U : Record + +> ; +> }; + +export type EmmettSystemsMap> = { +[K in keyof T]: EmmettSystem< +T[K] extends EmmettSystem ? U : Record + +> ; +> }; + +// const emmettComponent = , 'name'>>( +// name: string, +// config?: T, +// ) => { +// return { name, ...config } satisfies EmmettComponent; +// }; + +export type ComponentsOf = T extends { +components: infer M; +} +? M +: undefined; + +export function emmettComponent< +const Config extends { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +components?: Record>; +ports?: { +requires?: PortRequirements; +exposes?: PortRequirements; +}; +}, + +> ( +> name: string, +> config?: Config, +> ): { +> name: string; +> components: Config extends { components: infer C } ? C : undefined; +> ports: Config extends { ports: infer P } ? P : undefined; +> } { +> return { + + name, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + components: config?.components as any, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + ports: config?.ports as any, + +}; +} + +// ({ +// name, +// relationships: config?.relationships ? config.relationships : undefined, +// }) satisfies EmmettComponent as unknown as TReturn; + +function emmettContainer>( +name: string, +components?: T, +): EmmettContainer { +return { +name, +components, +}; +} + +const emmettSystem = >( +name: string, +containers?: T, +): EmmettSystem => ({ +name, +containers, +}); + +const emmettRelationship = < +Source extends AnyEmmettArchModule, +Target extends AnyEmmettArchModule, + +> ( +> source: Source, +> type: EmmettRelationshipType, +> target: Target, +> bundle?: ( + + target: Target['ports']['exposes'], + +) => Partial, +description?: string, +): EmmettRelationship => ({ +source: source.name, +target: target.name, +type, +description, +}); + +type ModuleBuilder = { +relationship: ( +target: Target, +type: EmmettRelationshipType, +description?: string, +) => EmmettRelationship; +}; + +export const moduleBuilder = ( +ctx: Source, +): ModuleBuilder => ({ +relationship: ( +target: Target, +type: EmmettRelationshipType, +description?: string, +) => emmettRelationship(ctx, type, target, undefined, description), +}); + +export const emmettArch = { +system: emmettSystem, +container: emmettContainer, +component: emmettComponent, +relationship: emmettRelationship, +}; + +and + +import { emmettArch } from '.'; + +const { component, container, relationship } = emmettArch; + +const query = +() => +(\_input: Input) => +Promise.resolve({} as Output); + +const getGuestByExternalId = (\_externalId: string): Promise => +Promise.resolve(\_externalId); + +const guests = component('guests', { +components: {}, +ports: { +requires: {}, +exposes: { +queries: { +getGuestByExternalId, +}, +}, +}, +}); + +const pricing = component('pricing'); + +const groupReservations = component('group-reservations'); + +const reservations = component('reservations', { +components: { groupReservations }, +ports: { +requires: { +guests: { +getGuestByExternalId: query(), +}, +}, +exposes: {}, +}, +}); + +const reservationsToGuests = relationship( +reservations, +'provides guest information to', +guests, +({ queries: { getGuestByExternalId } }) => ({ +guests: { +getGuestByExternalId, +}, +}), +); + +const hotelManagement = container('hotel-management', { +guests, +reservations, +pricing, +}); + +Where we could define container and pass dumboSchema to define modules inside e.g. postgresqlContainer (in the c4 model sense), + +Also how could I define also event store having multiple tables and functions, e.g. + +import type { SQLiteConnection } from '../../connection'; +import { +globalTag, +messagesTable, +streamsTable, +subscriptionsTable, +} from './typing'; + +export const sql = (sql: string) => sql; + +export const streamsTableSQL = sql( +`CREATE TABLE IF NOT EXISTS ${streamsTable.name}( + stream_id TEXT NOT NULL, + stream_position BIGINT NOT NULL DEFAULT 0, + partition TEXT NOT NULL DEFAULT '${globalTag}', + stream_type TEXT NOT NULL, + stream_metadata JSONB NOT NULL, + is_archived BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (stream_id, partition, is_archived), + UNIQUE (stream_id, partition, is_archived) + );`, +); + +export const messagesTableSQL = sql( +`CREATE TABLE IF NOT EXISTS ${messagesTable.name}( + stream_id TEXT NOT NULL, + stream_position BIGINT NOT NULL, + partition TEXT NOT NULL DEFAULT '${globalTag}', + message_kind CHAR(1) NOT NULL DEFAULT 'E', + message_data JSONB NOT NULL, + message_metadata JSONB NOT NULL, + message_schema_version TEXT NOT NULL, + message_type TEXT NOT NULL, + message_id TEXT NOT NULL, + is_archived BOOLEAN NOT NULL DEFAULT FALSE, + global_position INTEGER PRIMARY KEY, + created DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE (stream_id, stream_position, partition, is_archived) + ); +`, +); + +export const subscriptionsTableSQL = sql( +` CREATE TABLE IF NOT EXISTS ${subscriptionsTable.name}( + subscription_id TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + partition TEXT NOT NULL DEFAULT '${globalTag}', + last_processed_position BIGINT NOT NULL, + PRIMARY KEY (subscription_id, partition, version) + );`, +); + +export const schemaSQL: string[] = [ +streamsTableSQL, +messagesTableSQL, +subscriptionsTableSQL, +]; + +export const createEventStoreSchema = async ( +db: SQLiteConnection, +): Promise => { +for (const sql of schemaSQL) { +await db.command(sql); +} +}; + +export const appendToStreamSQL = rawSql( +`CREATE OR REPLACE FUNCTION emt_append_to_stream( +v_message_ids text[], +v_messages_data jsonb[], +v_messages_metadata jsonb[], +v_message_schema_versions text[], +v_message_types text[], +v_message_kinds text[], +v_stream_id text, +v_stream_type text, +v_expected_stream_position bigint DEFAULT NULL, +v_partition text DEFAULT emt_sanitize_name('default_partition') +) RETURNS TABLE ( +success boolean, +next_stream_position bigint, +global_positions bigint[], +transaction_id xid8 +) LANGUAGE plpgsql +AS $$ +DECLARE +v_next_stream_position bigint; +v_position bigint; +v_updated_rows int; +v_transaction_id xid8; +v_global_positions bigint[]; +BEGIN +v_transaction_id := pg_current_xact_id(); + + IF v_expected_stream_position IS NULL THEN + SELECT COALESCE( + (SELECT stream_position + FROM ${streamsTable.name} + WHERE stream_id = v_stream_id + AND partition = v_partition + AND is_archived = FALSE + LIMIT 1), + 0 + ) INTO v_expected_stream_position; + END IF; + + v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1); + + IF v_expected_stream_position = 0 THEN + INSERT INTO ${streamsTable.name} + (stream_id, stream_position, partition, stream_type, stream_metadata, is_archived) + VALUES + (v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE); + ELSE + UPDATE ${streamsTable.name} as s + SET stream_position = v_next_stream_position + WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE; + + get diagnostics v_updated_rows = row_count; + + IF v_updated_rows = 0 THEN + RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8; + RETURN; + END IF; + END IF; + + WITH ev AS ( + SELECT row_number() OVER () + v_expected_stream_position AS stream_position, + message_data, + message_metadata, + schema_version, + message_id, + message_type, + message_kind + FROM ( + SELECT * + FROM + unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds) + AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind) + ) AS message + ), + all_messages_insert AS ( + INSERT INTO ${messagesTable.name} + (stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id) + SELECT + v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id + FROM ev + RETURNING global_position + ) + SELECT + array_agg(global_position ORDER BY global_position) INTO v_global_positions + FROM + all_messages_insert; + + RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id; + +END; + +$$ +; +`, +); + +Potentially in different schemas (e.g. event store in one schema, read models using pongo in others. + +How could this be explicit? would we introduce some databaseFeatureSchemaComponent or databaseSchemaFeatureSchemaComponent? + +How to make it explicit that we're adding feature that can have multiple types of components but also be able to locate it in the specific schema or database? + +If I'm not clear enough, please ask for more. Ultrathink +[Show more] +I'll read the migration plan first to understand the context, then analyze how to integrate it with Pongo's schema system and the Architecture as Code concept. +Read MIGRATION_UNIFICATION_PLAN.md, index.ts +Now let me understand the question better by looking at some examples and clarifying a few points. I can see: +Dumbo's migration unification plan - A comprehensive system for managing database schemas with migrations, snapshots, and TypeScript definitions +Pongo's schema system - A higher-level abstraction focused on MongoDB-like collections within databases +Emmett Architecture as Code - A C4-model-inspired system for defining components, containers, and systems with ports and relationships +The user is asking about how to integrate these three concepts, specifically: +How to connect Dumbo's schema components with Pongo's database/collection abstractions +How to fit this into the Emmett Architecture as Code model +How to make it explicit when features span multiple schemas or databases +Dealing with the fact that Pongo can be "database or database per database schema" +Let me search for more context about event stores and how they're currently used: +Globpattern: "**/event*store*schema*.ts" +⎿ +Found 1 file +Grep "eventStoreSchema|event.*store.*component" +⎿ +1 line of output +Grep "databaseFeature|schemaFeature" +⎿ +1 line of output +Let me look for how dumbo schema components are currently structured: +Read dumboSchema.ts +Globpattern: "**/examples/**/*.ts" +⎿ +Found 1 file +Now I have a comprehensive understanding. Let me analyze the integration challenge and provide recommendations. The question involves several complex integration points: +Dumbo schema components - Low-level SQL schema definition with migrations +Pongo schema - MongoDB-like abstraction over SQL databases +Emmett Architecture as Code - C4-model components with ports and relationships +Physical deployment - Different database schemas or databases +Let me formulate my recommendations: +Claude’s Plan +Integration Plan: Dumbo + Pongo + Emmett Architecture as Code +Context Analysis +After reviewing the migration plan and existing code, I see three layers that need integration: +Dumbo Layer: Low-level database schema with SQL migrations (database → schema → table → column) +Pongo Layer: MongoDB-like abstraction (client → db → collection) +Emmett Layer: C4-model architecture components with ports/relationships +Key Challenge: "Pongo can be database OR database-per-schema" +This is the critical mapping issue: +PostgreSQL mode: One database, multiple schemas → Each Pongo DB maps to a PostgreSQL schema +SQLite mode: One database, no schema support → Each Pongo DB is just a logical grouping +Multi-tenant mode: Multiple databases → Each Pongo DB maps to a separate physical database +Recommended Architecture +1. Introduce databaseFeatureComponent +This bridges Dumbo's physical schema and Emmett's logical components: +export type DatabaseFeatureComponent< +NestedComponents extends Record> | undefined = undefined, +Requires extends PortRequirements | undefined = undefined, +Exposes extends PortRequirements | undefined = undefined, +> = EmmettComponent & { +// Physical database mapping +deployment: { + mode: 'database' | 'schema' | 'collection-prefix'; + database?: string; // Physical database name + schema?: string; // PostgreSQL schema name + prefix?: string; // For collection prefixing in shared DB +}; + +// Dumbo schema definition +dumboSchema: DatabaseSchemaComponent | DatabaseSchemaSchemaComponent; + +// Pongo schema definition +pongoSchema: PongoDbSchema; +}; +2. Event Store Example Integration +const eventStoreFeature = emmettArch.component('event-store', { +components: { + streams: emmettArch.component('streams-table'), + messages: emmettArch.component('messages-table'), + subscriptions: emmettArch.component('subscriptions-table'), + appendToStream: emmettArch.component('append-to-stream-function'), +}, +ports: { + exposes: { + commands: { + appendToStream: (/* ... */) => Promise.resolve(), + }, + queries: { + readStream: (/* ... */) => Promise.resolve([]), + }, + }, +}, +deployment: { + mode: 'schema', + database: 'main_db', + schema: 'event_store', +}, +dumboSchema: dumboDatabaseSchema('event_store', { + streams: dumboTable('streams', { /* ... */ }), + messages: dumboTable('messages', { /* ... */ }), + subscriptions: dumboTable('subscriptions', { /* ... */ }), +}), +pongoSchema: pongoSchema.db('event_store', { + streams: pongoSchema.collection('streams'), + messages: pongoSchema.collection('messages'), + subscriptions: pongoSchema.collection('subscriptions'), +}), +}); +3. Multi-Schema Container +For your hotel management example with separate schemas: +const hotelManagementContainer = emmettArch.container('hotel-management', { +// Event store in dedicated schema +eventStore: databaseFeatureComponent('event-store', { + deployment: { mode: 'schema', schema: 'event_store' }, + dumboSchema: eventStoreDumboSchema, + pongoSchema: eventStorePongoSchema, +}), + +// Read models in another schema +readModels: databaseFeatureComponent('read-models', { + deployment: { mode: 'schema', schema: 'read_models' }, + dumboSchema: readModelsDumboSchema, + pongoSchema: readModelsPongoSchema, + components: { + guests: emmettArch.component('guests'), + reservations: emmettArch.component('reservations'), + }, +}), +}); +4. Type-Safe Schema Navigation +Add helper to navigate from Emmett component to its schemas: +export const getComponentSchemas = ( +component: T +) => ({ +dumbo: component.dumboSchema, +pongo: component.pongoSchema, +deployment: component.deployment, + +// Get fully qualified table names +tables: () => { + const { mode, database, schema, prefix } = component.deployment; + // Return table references with correct qualifiers +}, +}); +5. Migration Integration +The orchestrator should understand deployment modes: +export class MultiTenantMigrationOrchestrator { +async deployFeature(feature: DatabaseFeatureComponent) { + const { mode, database, schema } = feature.deployment; + + switch (mode) { + case 'database': + // Deploy to separate database + await this.migrateDatabase(database!, feature.dumboSchema); + break; + case 'schema': + // Deploy to schema within database + await this.migrateSchema(database!, schema!, feature.dumboSchema); + break; + case 'collection-prefix': + // Just prefix collections, no migration needed + break; + } +} +} +Questions for Clarification +Naming: Should we call it databaseFeatureComponent or dataStoreComponent or persistenceComponent? +Nested features: If a feature has multiple schemas (e.g., event_store + projections), should each be a separate component or nested? +Cross-schema relationships: How should we express that read models depend on event store when they're in different schemas? +Collection prefixing: Should we support a "prefix" mode where multiple features share a schema but use prefixed collections? +Function/procedure components: Should stored procedures be first-class components or just part of table components? +Next Steps +Define DatabaseFeatureComponent type with deployment metadata +Extend Emmett arch builder with .databaseFeature() method +Create mapping helpers between Dumbo/Pongo/Emmett schemas +Update migration orchestrator to handle deployment modes +Add examples showing event store + read models in separate schemas +Stayed in plan mode +No, event store schema component should define all on their own. Potentially it can take pongoSchema or dumboSchema for inline projections. Read https://event-driven.io/en/emmett_projections_testing/ + +PongoSchema defines collections like mongo db, not eventstroe read: https://event-driven.io/en/pongo_strongly_typed_client/ + +I'd likesomeone to need to define all dumboSchema, pongo Schema etc. I'd like to be able to either wrap dumbo schema, or be able to pass those features to tell, e.g. like we have tables then including event store. or pongo collection. + +The obvious option is to include dumbodatabase schema into event store and pongoschema and build it with wrappers, but I'd like to avoid it if possible to just compose it as other schemaComponents +Ah, I understand much better now! Let me re-read those articles and rethink this. +Web Fetchhttps://event-driven.io/en/emmett_projections_testing/ +Fetched from https://event-driven.io/en/emmett_projections_testing/ +Web Fetchhttps://event-driven.io/en/pongo_strongly_typed_client/ +Fetched from https://event-driven.io/en/pongo_strongly_typed_client/ +Let me search for event store and projection examples in the codebase: +Grep "inline.*projection|projection.*inline" +⎿ +1 line of output +Grep "eventStore|event-store|EventStore" +⎿ +2 lines of output +Globpattern: "**/emmett**/projection*.ts" +⎿ +Found 1 file +Grep "class.*EventStore|function.*eventStore|const.*eventStore" +⎿ +1 line of output +Globpattern: "**/eventStore*.ts" +⎿ +Found 1 file +Globpattern: "**/event-store*.ts" +⎿ +Found 1 file +Let me look at the schema component structure to understand how components compose: +Read schemaComponent.ts, tableSchemaComponent.ts +If you don't know something then ask, don't hallucinate. I provided you a lot. Event Store is in emmett, not in this repo, that's why I've sent you code samples and articles... +If you need some more definition please ask. +Perfect! Now I understand the pattern. Let me think through what you're asking: Your key insights: +Event store should define its own schema components (tables, functions, etc.) +Pongo schema is for MongoDB-like collections, NOT for event store +You want to compose features like event store AS schema components, not wrap them +You want to avoid having to define dumbo/pongo schemas separately inside each feature +The pattern I see: +TableSchemaComponent extends SchemaComponent with table-specific data +It can be added to a DatabaseSchemaSchemaComponent as a component +You want something similar for "event store" or "pongo collections" +Let me propose a solution based on this understanding: +Proposed Solution: Feature Schema Components +1. EventStoreSchemaComponent (Dumbo side) +export type EventStoreURNType = 'sc:dumbo:event_store'; +export type EventStoreURN = `${EventStoreURNType}:${string}`; + +export type EventStoreSchemaComponent = SchemaComponent< +EventStoreURN, +Readonly<{ + eventStoreName: string; + tables: ReadonlyMap; + functions: ReadonlyMap; +}> +>; + +export const eventStoreSchemaComponent = ({ +eventStoreName, +...options +}: { +eventStoreName: string; +} & SchemaComponentOptions): EventStoreSchemaComponent => { +// Define the event store tables +const streamsTable = tableSchemaComponent({ + tableName: 'streams', + migrations: [/* SQL for streams table */] +}); + +const messagesTable = tableSchemaComponent({ + tableName: 'messages', + migrations: [/* SQL for messages table */] +}); + +const subscriptionsTable = tableSchemaComponent({ + tableName: 'subscriptions', + migrations: [/* SQL for subscriptions table */] +}); + +const appendFunction = functionSchemaComponent({ + functionName: 'emt_append_to_stream', + migrations: [/* SQL for function */] +}); + + const base = schemaComponent( + `sc:dumbo:event_store:${eventStoreName}` as EventStoreURN, + { + migrations: options.migrations ?? [], + components: [ + streamsTable, + messagesTable, + subscriptionsTable, + appendFunction, + ...(options.components ?? []), + ], + } +); + + return { + ...base, + eventStoreName, + get tables() { + return mapSchemaComponentsOfType( + base.components, + 'sc:dumbo:table', + (c) => c.tableName, + ); + }, + get functions() { + return mapSchemaComponentsOfType( + base.components, + 'sc:dumbo:function', + (c) => c.functionName, + ); + }, +}; +}; +2. PongoCollectionSetSchemaComponent (Pongo side) +For Pongo, you'd have a similar component that represents a set of collections: +export type PongoCollectionSetURNType = 'sc:pongo:collection_set'; +export type PongoCollectionSetURN = `${PongoCollectionSetURNType}:${string}`; + +export type PongoCollectionSetSchemaComponent = SchemaComponent< +PongoCollectionSetURN, +Readonly<{ + name: string; + pongoSchema: PongoDbSchema; + collections: ReadonlyMap; +}> +>; + +export const pongoCollectionSetSchemaComponent = ({ +name, +collections, +...options +}: { +name: string; +collections: Record; +} & SchemaComponentOptions): PongoCollectionSetSchemaComponent => { +const pongoDbSchema = pongoSchema.db(collections); + +const base = schemaComponent( + `sc:pongo:collection_set:${name}` as PongoCollectionSetURN, + { + migrations: options.migrations ?? [], + components: options.components ?? [], + } +); + + return { + ...base, + name, + pongoSchema: pongoDbSchema, + get collections() { + // Could map from pongoSchema.collections + return new Map(); + }, +}; +}; +3. Usage: Composing into DatabaseSchema +// Define event store as a schema component +const eventStore = eventStoreSchemaComponent({ +eventStoreName: 'default', +}); + +// Define read models as Pongo collections wrapped in dumbo tables +const readModelsSchema = dumboDatabaseSchema('read_models', { +guests: dumboTable('guests', { + columns: { + id: dumboColumn('id'), + name: dumboColumn('name'), + } +}), +reservations: dumboTable('reservations', { + columns: { + id: dumboColumn('id'), + guestId: dumboColumn('guest_id'), + } +}), +}); + +// Compose them together +const hotelDatabase = dumboDatabase('hotel', { +event_store: dumboDatabaseSchema('event_store', { + // Add event store as components! Tables are automatically included +}).addComponent(eventStore), // <-- Event store adds its tables + +read_models: readModelsSchema, +}); +Wait, that's still awkward. Let me reconsider... +Better Approach: Event Store IS a DatabaseSchemaSchemaComponent +export const eventStoreSchema = ( +schemaName: string = 'event_store', +options?: SchemaComponentOptions +): DatabaseSchemaSchemaComponent => { +return dumboDatabaseSchema(schemaName, { + streams: dumboTable('streams', { + migrations: [sqlMigration('create_streams', [streamsTableSQL])], + }), + messages: dumboTable('messages', { + migrations: [sqlMigration('create_messages', [messagesTableSQL])], + }), + subscriptions: dumboTable('subscriptions', { + migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], + }), +}, options); +}; + +// Then you can use it directly: +const hotelDatabase = dumboDatabase('hotel', { +event_store: eventStoreSchema('event_store'), +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), +}), +}); +But you also want inline projections... Let me think about that: +With Inline Projections +export const eventStoreSchema = ({ +schemaName = 'event_store', +inlineProjections = {}, +...options +}: { +schemaName?: string; +inlineProjections?: Record; +} & SchemaComponentOptions = {}): DatabaseSchemaSchemaComponent => { +const coreEventStoreTables = { + streams: dumboTable('streams', { + migrations: [sqlMigration('create_streams', [streamsTableSQL])], + }), + messages: dumboTable('messages', { + migrations: [sqlMigration('create_messages', [messagesTableSQL])], + }), + subscriptions: dumboTable('subscriptions', { + migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], + }), +}; + +// Convert inline projections to dumbo tables if needed +const projectionTables: Record = {}; +for (const [name, projection] of Object.entries(inlineProjections)) { + if ('tableName' in projection) { + // It's already a TableSchemaComponent + projectionTables[name] = projection; + } else { + // It's a PongoCollectionSchema, create a table for it + projectionTables[name] = dumboTable(projection.name, {}); + } +} + +return dumboDatabaseSchema(schemaName, { + ...coreEventStoreTables, + ...projectionTables, +}, options); +}; + +// Usage: +const hotelDatabase = dumboDatabase('hotel', { +event_store: eventStoreSchema({ + schemaName: 'event_store', + inlineProjections: { + guest_summary: pongoSchema.collection('guest_summary'), + reservation_summary: dumboTable('reservation_summary', { /* ... */ }), + }, +}), +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), +}), +}); +For Emmett Architecture Integration +Now, how does this relate to Emmett components? +const eventStoreComponent = emmettArch.component('event-store', { +ports: { + exposes: { + commands: { + appendToStream: (/* ... */) => Promise.resolve(), + }, + }, +}, +}); + +const guestsComponent = emmettArch.component('guests', { +ports: { + exposes: { + queries: { + getGuestByExternalId: query(), + }, + }, +}, +}); + +// The container ties Emmett components to database schemas +const hotelManagementContainer = emmettArch.container('hotel-management', { +// Emmett components +eventStore: eventStoreComponent, +guests: guestsComponent, +reservations: reservationsComponent, +}); + +// Separately, define the database schema +const hotelDatabaseSchema = dumboDatabase('hotel', { +event_store: eventStoreSchema({ schemaName: 'event_store' }), +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), +}), +}); +But how do we link them? +Key Questions: +Should Emmett components contain schema definitions? Or should they be separate? +Should we have a mapping layer that says "event-store component uses event_store schema"? +Do you want deployment flexibility where the same Emmett component can be deployed to different schemas/databases? +For the "explicit location" problem: Do you want something like: +// Query that knows it's in event_store schema +const streams = database.schema('event_store').table('streams'); + +// Or from Emmett component perspective +const component = container.components.eventStore; +const schema = getSchemaFor(component); // Returns event_store schema +Let me ask some clarifying questions to nail down the design: Clarifying Questions: +Separation of concerns: Should Emmett components (business logic) be completely separate from dumbo schemas (persistence), or should they reference each other? +Event store as primitive: Should eventStoreSchema() be a first-class schema component type (like dumboTable), or just a factory function that creates a DatabaseSchemaSchemaComponent? +Inline projections: When you say "event store can take pongoSchema for inline projections", do you mean: +a) Event store schema should accept PongoCollectionSchema and convert them to tables +b) Event store runtime should support Pongo query API on projection tables +c) Both? +Multi-schema deployment: If you have event_store in one schema and read_models in another, how should the Emmett component know which schema to query? Should there be a deployment config that maps components to schemas? +Function/Stored Procedures: Should SQL functions (like emt_append_to_stream) be first-class schema components, or just part of table/schema migrations? +Please clarify these points so I can provide the right solution! +You're absolutely right - I apologize for making assumptions. Let me ask clarifying questions based on what you've actually provided: +Clarifying Questions +Based on the code samples you provided and the migration plan, I need to understand: +1. Event Store Structure +From your SQL example, I can see the event store has: +Tables: streams, messages, subscriptions +Functions: emt_append_to_stream (PostgreSQL stored procedure) +These are defined via raw SQL (streamsTableSQL, messagesTableSQL, etc.) +Question: Should event store be represented as: +Option A: A factory function that returns a DatabaseSchemaSchemaComponent with all tables pre-configured? +const eventStore = eventStoreSchema('event_store'); // Returns DatabaseSchemaSchemaComponent +Option B: A new schema component type (like EventStoreSchemaComponent extends SchemaComponent)? +Option C: Just a collection of table components that you manually compose? +2. Inline Projections +You mentioned "event store can take pongoSchema or dumboSchema for inline projections" Question: What does this mean in practice? +Do inline projections create additional tables in the same schema as the event store? +Should I be able to do something like: +const eventStore = eventStoreSchema('event_store', { +inlineProjections: { + guestSummary: pongoSchema.collection('guest_summary'), + // This creates a 'guest_summary' table in the event_store schema? +} +}); +3. Pongo vs Dumbo Relationship +From pongo/src/core/schema/index.ts: +Pongo schema defines client -> db -> collections (MongoDB-like) +It's a logical abstraction over the physical database +From the migration plan: +Dumbo schema defines database -> schema -> table -> column (SQL) +It's the physical structure with migrations +Question: How should these relate? +Can a single Pongo db map to multiple Dumbo schemas? +If I have event_store in one schema and read_models in another, do I create: +Two separate Pongo dbs? +One Pongo db that somehow spans both schemas? +4. Stored Procedures / Functions +Your emt_append_to_stream example is a PostgreSQL function. Question: Should functions be: +Option A: Part of table migrations (stored in table component)? +Option B: A new FunctionSchemaComponent type (like TableSchemaComponent)? +Option C: Part of schema-level migrations (stored in DatabaseSchemaSchemaComponent)? +5. Emmett Component Integration +You showed Emmett Architecture as Code with components, containers, ports. Question: What's the relationship between: +Emmett component (business logic with ports) +Dumbo DatabaseSchemaSchemaComponent (physical schema) +Pongo PongoDbSchema (logical collections) +Should they: +Option A: Be completely separate (define them independently, link via configuration)? +Option B: Have explicit references (Emmett component has .schema property)? +Option C: Use a mapping/registry pattern? +6. Multiple Schemas in One Database +Your question mentions "pongo database could be either database or database per database schema" Question: Can you explain this with a concrete example? +// Scenario 1: PostgreSQL with multiple schemas +const db = pongoClient.db('hotel'); // Maps to PostgreSQL database 'hotel' +// This database has schemas: event_store, read_models, analytics + +// Scenario 2: SQLite (no schemas) +const db = pongoClient.db('hotel'); // Maps to SQLite database 'hotel' +// No schema concept, all tables in one namespace + +// How should the schema definition work for both? +7. "Explicit Location" Requirement +You said: "How to make it explicit that we're adding feature that can have multiple types of components but also be able to locate it in the specific schema or database?" Question: What does "locate it" mean? +At definition time: "This feature's tables go in schema X"? +At runtime: "Query this component and know which schema it's in"? +At architecture documentation time: "Show which schema each component uses"? +8. Composition Pattern +You said: "I'd like to just compose it as other schemaComponents" Question: What does the ideal composition look like to you? +// Example A: Flat composition +const hotelDb = dumboDatabase('hotel', { +event_store: eventStoreSchema(), +read_models: readModelsSchema(), +}); + +// Example B: Nested composition +const hotelDb = dumboDatabase('hotel') +.addSchema(eventStoreSchema()) +.addSchema(readModelsSchema()); + +// Example C: Component-based +const hotelDb = dumboDatabase('hotel', { +features: { + eventStore: eventStoreFeature, // This internally creates schemas + readModels: readModelsFeature, +} +}); +Please clarify these points and I'll design the right solution based on what you actually need, not assumptions! +Thank you for your patience. Let me ask the essential questions I need to understand your requirements: +Essential Questions +1. Event Store Package Location & API +You mentioned "Event Store is in emmett, not in this repo" and provided SQL samples. Question: In the Emmett package, does the event store: +Export a function/class that creates the schema definition? +Provide just the SQL strings (like your streamsTableSQL, messagesTableSQL examples)? +Provide a complete schema component I can import and use? +What I need to know: If I want to use event store in this codebase, what exactly do I import from Emmett and how do I turn it into a Dumbo schema component? +2. The Core Integration Problem +You want to avoid wrapping dumbo/pongo schemas inside feature definitions, but still compose them. Question: Starting from your event store SQL example with streams, messages, subscriptions tables and the emt_append_to_stream function - if I want to: +Use this as a Dumbo schema component (for migrations) +Also expose it as a Pongo schema (for querying) +Include it in an Emmett architecture component (for documentation/ports) +Deploy it to a specific PostgreSQL schema (e.g., event_store schema) +What should the code look like? Can you show me a pseudocode example of how you envision defining this? +3. Inline Projections Clarification +You said event store "can take pongoSchema or dumboSchema for inline projections" Question: When you define an inline projection: +// Pseudocode - is this the right direction? +eventStore({ +inlineProjections: { + guestSummary: ??? // What goes here? +} +}) +Does guestSummary become a table in the same PostgreSQL schema as streams/messages? +Is it a Pongo collection definition that gets converted to a table? +Is it a Dumbo table definition? +Something else? +4. Physical Deployment Mapping +Regarding "pongo database could be either database or database per database schema": Question: In these two scenarios, how should the schemas be defined? Scenario A - PostgreSQL (one DB, multiple schemas): +-- Physical structure: +-- Database: hotel_db +-- Schema: event_store (tables: streams, messages, subscriptions) +-- Schema: read_models (tables: guests, reservations) +Scenario B - SQLite (one DB, no schema support): +-- Physical structure: +-- Database: hotel.db +-- Tables: streams, messages, subscriptions, guests, reservations +Should the Dumbo/Pongo schema definition be: +The same for both, with deployment config handling the difference? +Different for each database type? +Something else? +5. Function/Stored Procedure Components +Your emt_append_to_stream example is a PostgreSQL function. Question: Should I create a new FunctionSchemaComponent type (similar to TableSchemaComponent), or should functions just be SQL migrations attached to schema/table components? If you want FunctionSchemaComponent, should it be: +const appendFunction = dumboFunction('emt_append_to_stream', { +migrations: [sqlMigration('create_append', [appendToStreamSQL])], +}); + +// Then added to schema +const eventStoreSchema = dumboDatabaseSchema('event_store', { +tables: { streams, messages, subscriptions }, +functions: { appendToStream: appendFunction }, // ??? +}); +6. Emmett Component Connection +Looking at your Emmett Architecture example with emmettArch.component(), emmettArch.container(): Question: Should Emmett components: +Option A: Have no knowledge of Dumbo/Pongo schemas (purely architectural documentation)? +Option B: Reference which schema they use (e.g., eventStoreComponent.schema = eventStoreDumboSchema)? +Option C: Be created FROM schema definitions (e.g., emmettArch.component.fromSchema(eventStoreDumboSchema))? +7. Concrete Use Case +Can you provide a concrete example of what you want to build? For instance: +// I want to define: +// 1. Event store in its own PostgreSQL schema +// 2. Guest read model in another schema +// 3. Both accessible via Pongo API +// 4. Both documented in Emmett architecture +// 5. The event store should have an inline projection for guest summaries + +// What should this code look like? +const ??? = ??? +If you can show me the desired API/syntax you want to write, I can design the implementation to support it. +Please answer whichever questions are relevant to clarify your vision. I want to understand what you're trying to achieve before proposing a solution. +I don't like thaat as some just want to keep event store in the same schema as read model. + +The idea in general is to be able to also run migration together for database that contain both event store and read modls. And yes, by default I don't want user to know internal structure of event store, same for pongo collections. they should be opaque, but in the future give some customisation. +Ah! Now I understand much better. Let me clarify what I'm hearing: +What I Understand Now +Flexibility in deployment: Event store and read models can be in: +Same schema (simpler deployments) +Different schemas (separation of concerns) +User's choice, not forced +Opaque internals: Users shouldn't need to know that event store has streams, messages, subscriptions tables - that's an implementation detail +Run migrations together: Whether event store is in the same schema or different, migrations should run together as one cohesive database setup +Future customization: Later, allow power users to customize event store internals, but not required +Key Question: What Does the API Look Like? +Let me propose what I think you want and you tell me if I'm on the right track: +Scenario 1: Event Store + Read Models in SAME schema +// User just wants everything in 'public' schema +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + // Event store tables are added transparently + ...eventStoreSchemaComponents(), // Returns { streams: table, messages: table, ... } + + // User's read model tables + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), +}) +}); +Scenario 2: Event Store + Read Models in DIFFERENT schemas +// User wants separation +const hotelDb = dumboDatabase('hotel', { +event_store: eventStoreSchema('event_store'), // Opaque, returns DatabaseSchemaSchemaComponent + +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), +}) +}); +Scenario 3: With Inline Projections +const hotelDb = dumboDatabase('hotel', { +event_store: eventStoreSchema('event_store', { + // Inline projections become tables in the SAME schema as event store + inlineProjections: { + guest_summary: pongoSchema.collection('guest_summary'), // Creates guest_summary table + } +}), + +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), +}) +}); +Critical Questions: +Q1: Event Store Schema Component API +Should eventStoreSchema() return a DatabaseSchemaSchemaComponent that contains all the tables internally? +// Option A: Opaque schema component +export const eventStoreSchema = ( +schemaName: string, +options?: { inlineProjections?: Record } +): DatabaseSchemaSchemaComponent => { +// Internally creates streams, messages, subscriptions tables +// User doesn't see them +}; + +// Option B: Spread/merge pattern +export const eventStoreSchemaComponents = (): Record => ({ +streams: dumboTable('streams', { migrations: [streamsSQL] }), +messages: dumboTable('messages', { migrations: [messagesSQL] }), +subscriptions: dumboTable('subscriptions', { migrations: [subscriptionsSQL] }), +}); +Which pattern do you prefer? +Q2: Pongo Schema Relationship +If event store and read models are in the same schema, how does Pongo schema map? +// If both in 'public' schema: +const pongoDb = pongoSchema.db('hotel', { +// Do we expose event store collections? +streams: pongoSchema.collection('streams'), // ??? +messages: pongoSchema.collection('messages'), // ??? + +// Or only read models? +guests: pongoSchema.collection('guests'), +reservations: pongoSchema.collection('reservations'), +}); +Should event store tables be exposed via Pongo, or hidden? +Q3: Migration Composition +When running migrations, should event store migrations: +Be part of the normal migration flow (just more tables)? +Run separately/first (event store setup, then read models)? +Be configurable? +Q4: Stored Functions +The emt_append_to_stream PostgreSQL function - should it be: +// Option A: Part of schema migrations +const eventStoreSchema = (schemaName: string) => +dumboDatabaseSchema(schemaName, { + streams: dumboTable('streams', { /* ... */ }), + messages: dumboTable('messages', { /* ... */ }), +}, { + migrations: [ + sqlMigration('create_functions', [appendToStreamSQL]) + ] +}); + +// Option B: First-class function component +const eventStoreSchema = (schemaName: string) => +dumboDatabaseSchema(schemaName, { + streams: dumboTable('streams', { /* ... */ }), + appendToStream: dumboFunction('emt_append_to_stream', { /* ... */ }), +}); +Which approach? +Please tell me: +Which API scenarios (1, 2, 3 above) match what you want? +Which of the options for each question is closer to your vision? +Any corrections to my understanding? +This will help me propose the right design! +I don't like this ...eventStoreSchemaComponents(), - I'd prefer to just use eventStoreSchema(), which could take also inline projection with references to pongo collection, still, pongo collection could be also define inside schema just like we do for table, and pongoDatabase both as database and schema. + +We should be able to make eventStoreSchema and pongo related shcemas to extend some base feature schema component, and then detect it, right or not? +YES! Now I get it! Let me confirm my understanding: +What You Want +1. Event Store as Opaque Component +// User just uses it, doesn't see internal tables +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), // NOT spreading, just add it as a component + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), +}) +}); +2. Feature Schema Component Pattern +// Base type that event store, pongo collections, etc. extend +export type FeatureSchemaComponent = SchemaComponent< +string, +{ + featureType: 'event_store' | 'pongo_collections' | 'custom'; +} +>; + +// Event store extends this +export type EventStoreSchemaComponent = SchemaComponent< +'sc:dumbo:event_store', +{ + featureType: 'event_store'; + eventStoreName: string; + // Internal: tables, functions - user doesn't see + internalTables: Map; +} +>; + +// Pongo collection set extends this +export type PongoCollectionSetSchemaComponent = SchemaComponent< +'sc:dumbo:pongo_collections', +{ + featureType: 'pongo_collections'; + pongoSchema: PongoDbSchema; +} +>; +3. Detection & Composition +// When adding to schema, detect what type it is +dumboDatabaseSchema('public', { +// This is detected as EventStoreSchemaComponent +eventStore: eventStoreSchema(), + +// This is detected as TableSchemaComponent +guests: dumboTable('guests', { /* ... */ }), + +// This is detected as PongoCollectionSetSchemaComponent +readModels: pongoCollectionsSchema({ + guests: pongoSchema.collection('guests'), + reservations: pongoSchema.collection('reservations'), +}), +}); +4. Inline Projections with Pongo Collections +const hotelDb = dumboDatabase('hotel', { +event_store: dumboDatabaseSchema('event_store', { + // Event store with inline projections + eventStore: eventStoreSchema({ + inlineProjections: { + // Reference to pongo collection - creates table automatically + guestSummary: pongoSchema.collection('guest_summary'), + } + }), +}), + +read_models: dumboDatabaseSchema('read_models', { + // Or define pongo collections directly in schema + readModels: pongoCollectionsSchema({ + guests: pongoSchema.collection('guests'), + reservations: pongoSchema.collection('reservations'), + }), +}), +}); +Key Questions to Confirm: +Q1: Feature Schema Component Base +Should there be a common base or marker for "feature components" that: +Contain multiple internal components (tables, functions, etc.) +Can be detected via isSchemaComponentOfType(comp, 'feature') or similar +Are opaque (internal structure hidden from user) +export const isFeatureSchemaComponent = (comp: AnySchemaComponent): boolean => { +return comp.schemaComponentKey.startsWith('sc:dumbo:feature:'); +}; +Q2: Pongo Collections as Schema Component +Should pongoCollectionsSchema() create tables automatically? +// This creates: +// - A schema component +// - Dumbo tables for each collection (for migrations) +// - Pongo schema for querying +const readModels = pongoCollectionsSchema({ +guests: pongoSchema.collection('guests'), +reservations: pongoSchema.collection('reservations'), +}); + +// Internally generates: +// - dumboTable('guests', {}) +// - dumboTable('reservations', {}) +// - pongoSchema.db({ guests, reservations }) +Is this correct? +Q3: Same Schema Flexibility +All of these should work, right? +// A: Everything in one schema +dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), + guests: dumboTable('guests', {}), +}) +}); + +// B: Separated schemas +dumboDatabase('hotel', { +event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), +}), +read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', {}), +}) +}); + +// C: Event store + pongo collections in same schema +dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), + readModels: pongoCollectionsSchema({ + guests: pongoSchema.collection('guests'), + }), +}) +}); +Q4: Detection in Migration Orchestrator +When collecting migrations, detect feature components: +// In migration orchestrator +for (const component of schema.components.values()) { +if (isEventStoreComponent(component)) { + // Handle event store migrations specially if needed +} else if (isPongoCollectionsComponent(component)) { + // Generate table migrations from pongo collections +} else if (isTableComponent(component)) { + // Standard table migration +} +} +Is this the right approach? +Am I understanding correctly now? The key insight is: +Feature components (event store, pongo collections) are first-class schema components +They compose like tables do +They're detected by type checking +They encapsulate internal structure +Should I proceed with this design? +For user feature should be opaque, they don't need to know how sausages are made, still, they should be visible later in generated typed system and added to the dumbo schema definition so they're strongly typed and also that migrations works fine. + +Ad q3. + +A looks weird as it suggests that eventStore is table, which is not. +For user feature should be opaque, they don't need to know how sausages are made, still, they should be visible later in generated typed system and added to the dumbo schema definition so they're strongly typed and also that migrations works fine. + +Ad q3. + +A looks weird as it suggests that eventStore is table, which is not. +Perfect! This clarifies a lot. Let me refine my understanding: +Key Insights +1. Feature Components are NOT Tables +You're right - in scenario A, putting eventStore: eventStoreSchema() alongside tables is confusing because it's not a table. Better API: +// Option A-revised: Feature at schema level +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + features: { + eventStore: eventStoreSchema(), + }, + tables: { + guests: dumboTable('guests', {}), + reservations: dumboTable('reservations', {}), + } +}) +}); + +// OR Option A-alternative: Feature creates its own implicit schema content +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public') + .withFeature(eventStoreSchema()) + .withTable(dumboTable('guests', {})) +}); + +// OR Option A-simpler: Features and tables mixed but clearly typed +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), // Type system knows this is FeatureSchemaComponent + guests: dumboTable('guests', {}), // Type system knows this is TableSchemaComponent +}) +}); +Question: Which API feels right to you? I'm leaning toward the last one (A-simpler) because: +Type system prevents confusion +Simpler API +Components compose uniformly +2. Strongly Typed Generated System +You want the generated types to expose the feature's tables: +// After running schema generation from: +const hotelDb = dumboDatabase('hotel', { +public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), + guests: dumboTable('guests', {}), +}) +}); + +// Generated types should include: +export type HotelSchema = { +public: { + // Event store tables exposed (even though user didn't define them) + streams: { stream_id: string, stream_position: number, ... }, + messages: { stream_id: string, message_data: unknown, ... }, + subscriptions: { subscription_id: string, ... }, + + // User's tables + guests: { id: string, name: string, ... }, +} +}; +Question: Should the generated types expose event store internals, or keep them hidden? +3. Pongo Schema Generation +For Pongo collections defined via pongoCollectionsSchema(): +const hotelDb = dumboDatabase('hotel', { +read_models: dumboDatabaseSchema('read_models', { + collections: pongoCollectionsSchema({ + guests: pongoSchema.collection('guests'), + reservations: pongoSchema.collection('reservations'), + }), +}) +}); + +// Should generate: +// 1. Dumbo tables: guests, reservations +// 2. Pongo schema: PongoDbSchema with those collections +// 3. Migrations: CREATE TABLE for each collection +Proposed Architecture +Feature Schema Component Base +// Marker interface +export type FeatureSchemaComponent< +URN extends string = string, +AdditionalData extends Record = Record +> = SchemaComponent; // Hidden from user +}>; + +// Type guard +export const isFeatureSchemaComponent = ( +comp: AnySchemaComponent +): comp is FeatureSchemaComponent => { +return '__featureMarker' in comp && comp.__featureMarker === true; +}; +Event Store Schema Component +export type EventStoreSchemaComponent = FeatureSchemaComponent< +'sc:dumbo:feature:event_store', +{ + eventStoreName?: string; + inlineProjections?: Record; +} +>; + +export const eventStoreSchema = ( +options?: { + inlineProjections?: Record; +} +): EventStoreSchemaComponent => { +// Create internal tables +const streams = dumboTable('streams', { + migrations: [sqlMigration('create_streams', [streamsTableSQL])] +}); +const messages = dumboTable('messages', { + migrations: [sqlMigration('create_messages', [messagesTableSQL])] +}); +const subscriptions = dumboTable('subscriptions', { + migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])] +}); + +// Handle inline projections +const projectionTables = Object.entries(options?.inlineProjections ?? {}).map( + ([name, collectionSchema]) => dumboTable(collectionSchema.name, {}) +); + +const allComponents = [streams, messages, subscriptions, ...projectionTables]; + +const base = schemaComponent('sc:dumbo:feature:event_store', { + components: allComponents, +}); + +return { + ...base, + __featureMarker: true as const, + eventStoreName: 'event_store', + inlineProjections: options?.inlineProjections, + internalComponents: new Map(allComponents.map(c => [c.schemaComponentKey, c])), +}; +}; +Pongo Collections Schema Component +export type PongoCollectionsSchemaComponent = FeatureSchemaComponent< +'sc:dumbo:feature:pongo_collections', +{ + pongoSchema: PongoDbSchema; +} +>; + +export const pongoCollectionsSchema = ( +collections: Record +): PongoCollectionsSchemaComponent => { +// Create dumbo table for each pongo collection +const tables = Object.entries(collections).map(([_name, collectionSchema]) => + dumboTable(collectionSchema.name, { + // Auto-generate basic table structure for Pongo + // Could add _id, document JSONB columns, etc. + }) +); + +const base = schemaComponent('sc:dumbo:feature:pongo_collections', { + components: tables, +}); + +return { + ...base, + __featureMarker: true as const, + pongoSchema: pongoSchema.db(collections), + internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), +}; +}; +Migration Collection +// In migration orchestrator +export const collectAllMigrations = ( +database: DatabaseSchemaComponent, +): SQLMigration[] => { +const migrations: SQLMigration[] = []; + +for (const schema of database.schemas.values()) { + for (const component of schema.components.values()) { + if (isFeatureSchemaComponent(component)) { + // Feature component: extract internal components + for (const internalComp of component.internalComponents.values()) { + if (isTableComponent(internalComp)) { + migrations.push(...internalComp.migrations); + } + } + } else if (isTableComponent(component)) { + // Regular table + migrations.push(...component.migrations); + } + } +} + +return migrations; +}; +Final Clarifying Questions +Q1: Component Placement +Which feels most natural? +// A: Mixed (type system differentiates) +dumboDatabaseSchema('public', { +eventStore: eventStoreSchema(), +guests: dumboTable('guests', {}), +}) + +// B: Separated +dumboDatabaseSchema('public', { +features: { eventStore: eventStoreSchema() }, +tables: { guests: dumboTable('guests', {}) }, +}) + +// C: Builder pattern +dumboDatabaseSchema('public') +.withFeature(eventStoreSchema()) +.withTable(dumboTable('guests', {})) +Q2: Generated Types Visibility +Should generated types expose event store internals? +// Option A: Expose everything +type Schema = { +public: { + streams: { ... }, // From event store + messages: { ... }, // From event store + guests: { ... }, // From user +} +} + +// Option B: Keep features opaque +type Schema = { +public: { + guests: { ... }, // Only user-defined tables +} +} +Q3: Pongo Collections Table Structure +When pongoCollectionsSchema() creates tables, what should the table structure be? +// Option A: Document store (MongoDB-like) +CREATE TABLE guests ( +_id TEXT PRIMARY KEY, +document JSONB NOT NULL +) + +// Option B: Let user define structure +pongoCollectionsSchema({ +guests: pongoSchema.collection('guests', { + tableDefinition: dumboTable('guests', { + columns: { id, name, email, ... } + }) +}) +}) + +// Option C: Infer from Pongo usage later +Please confirm these choices and I'll build the complete solution! +$$ diff --git a/src/packages/dumbo/src/core/schema/schema_feature.md b/src/packages/dumbo/src/core/schema/schema_feature.md new file mode 100644 index 00000000..afa5b776 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/schema_feature.md @@ -0,0 +1,1197 @@ +# Schema Feature Component Design + +## Overview + +This document defines the design for **Feature Schema Components** - a composable, opaque abstraction layer for complex database features like event stores, Pongo collections, and custom application features that can be integrated seamlessly into Dumbo's schema system. + +## Problem Statement + +Current Dumbo schema components (`database`, `schema`, `table`, `column`, `index`) work well for explicit table definitions, but pose challenges for: + +1. **Complex Features**: Features like event stores have multiple tables, functions, and internal structure that users shouldn't need to understand +2. **Framework Integration**: Pongo collections need both Dumbo tables (for migrations) and Pongo schema (for querying) +3. **Multi-Database Support**: Same schema definition should work across PostgreSQL (with schemas), SQLite (without schemas), and other databases +4. **Multi-Tenancy**: Need flexible sharding strategies applicable at different levels (table, schema, database) +5. **System Metadata**: Migration tracking and schema introspection tables need consistent, configurable placement + +## Design Principles + +1. **Opaque by Default**: Users don't need to know internal structure of features (e.g., event store tables) +2. **Composable**: Features compose like other schema components +3. **Strongly Typed**: Generated types expose internal structure for type-safe queries +4. **Database Agnostic**: Schema definitions are portable; migration generation is database-specific +5. **Extensible**: Generic sharding mechanism supports multi-tenancy and custom partitioning strategies +6. **Explicit When Needed**: Power users can customize feature internals and placement + +## Core Concepts + +### 1. Feature Schema Component + +A feature is an opaque schema component that encapsulates internal components (tables, functions, etc.). + +```typescript +// Base marker interface for all features +export type FeatureSchemaComponent< + URN extends string = string, + AdditionalData extends Record = Record +> = SchemaComponent; // Hidden from user API +}>; + +// Type guard +export const isFeatureSchemaComponent = ( + comp: AnySchemaComponent +): comp is FeatureSchemaComponent => { + return '__featureMarker' in comp && comp.__featureMarker === true; +}; +``` + +### 2. Event Store Schema Component + +Event store as a first-class feature component. + +```typescript +export type EventStoreSchemaComponent = FeatureSchemaComponent< + 'sc:dumbo:feature:event_store', + { + eventStoreName?: string; + inlineProjections?: Record; + } +>; + +export const eventStoreSchema = ( + options?: { + inlineProjections?: Record; + } +): EventStoreSchemaComponent => { + // Create internal tables (opaque to user) + const streams = dumboTable('streams', { + migrations: [sqlMigration('create_streams', [streamsTableSQL])] + }); + + const messages = dumboTable('messages', { + migrations: [sqlMigration('create_messages', [messagesTableSQL])] + }); + + const subscriptions = dumboTable('subscriptions', { + migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])] + }); + + // Create functions + const appendFunction = dumboFunction('emt_append_to_stream', { + migrations: [sqlMigration('create_append_function', [appendToStreamSQL])] + }); + + // Handle inline projections (create tables from Pongo collections) + const projectionTables = Object.entries(options?.inlineProjections ?? {}).map( + ([name, collectionSchema]) => dumboTable(collectionSchema.name, { + // Auto-generate table structure for Pongo collection + migrations: [sqlMigration(`create_projection_${name}`, [ + // Generate CREATE TABLE with _id and document columns + ])] + }) + ); + + const allComponents = [ + streams, + messages, + subscriptions, + appendFunction, + ...projectionTables + ]; + + const base = schemaComponent('sc:dumbo:feature:event_store', { + components: allComponents, + }); + + return { + ...base, + __featureMarker: true as const, + eventStoreName: 'event_store', + inlineProjections: options?.inlineProjections, + internalComponents: new Map(allComponents.map(c => [c.schemaComponentKey, c])), + }; +}; +``` + +### 3. Pongo Collections Schema Component + +Bridges Pongo's collection abstraction with Dumbo's table-based migrations. + +```typescript +export type PongoCollectionsSchemaComponent = FeatureSchemaComponent< + 'sc:dumbo:feature:pongo_collections', + { + pongoSchema: PongoDbSchema; + } +>; + +export const pongoCollectionsSchema = ( + collections: Record +): PongoCollectionsSchemaComponent => { + // Create Dumbo table for each Pongo collection + const tables = Object.entries(collections).map(([_name, collectionSchema]) => + dumboTable(collectionSchema.name, { + migrations: [sqlMigration(`create_collection_${collectionSchema.name}`, [ + // Auto-generate table structure: + // CREATE TABLE {name} ( + // _id TEXT PRIMARY KEY, + // document JSONB NOT NULL, + // created_at TIMESTAMP DEFAULT NOW(), + // updated_at TIMESTAMP DEFAULT NOW() + // ) + ])] + }) + ); + + const base = schemaComponent('sc:dumbo:feature:pongo_collections', { + components: tables, + }); + + return { + ...base, + __featureMarker: true as const, + pongoSchema: pongoSchema.db(collections), + internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), + }; +}; +``` + +### 4. System Schema Component + +Contains framework metadata tables (migrations, pongo metadata, schema introspection). + +```typescript +export type SystemSchemaComponent = FeatureSchemaComponent< + 'sc:dumbo:feature:system', + { + systemTables: { + migrationTracking: boolean; + pongoMetadata: boolean; + schemaIntrospection: boolean; + }; + } +>; + +export function systemSchema(): SystemSchemaComponent; +export function systemSchema(schemaName: string): DatabaseSchemaSchemaComponent; +export function systemSchema(options: { + migrationTracking?: boolean; + pongoMetadata?: boolean; + schemaIntrospection?: boolean; +}): SystemSchemaComponent; +export function systemSchema( + nameOrOptions?: string | { + migrationTracking?: boolean; + pongoMetadata?: boolean; + schemaIntrospection?: boolean; + } +): SystemSchemaComponent | DatabaseSchemaSchemaComponent { + const options = typeof nameOrOptions === 'string' + ? { schemaName: nameOrOptions } + : nameOrOptions; + + const tables: TableSchemaComponent[] = []; + + if (options?.migrationTracking !== false) { + tables.push(dumboTable('__migrations', { + migrations: [sqlMigration('create_migrations_table', [ + SQL`CREATE TABLE IF NOT EXISTS __migrations ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + applied_at TIMESTAMP DEFAULT NOW() + )` + ])] + })); + } + + if (options?.pongoMetadata !== false) { + tables.push(dumboTable('__pongo_collections', { + migrations: [sqlMigration('create_pongo_metadata_table', [ + SQL`CREATE TABLE IF NOT EXISTS __pongo_collections ( + collection_name TEXT PRIMARY KEY, + json_schema JSONB, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )` + ])] + })); + } + + if (options?.schemaIntrospection !== false) { + tables.push(dumboTable('__schema_metadata', { + migrations: [sqlMigration('create_schema_metadata_table', [ + SQL`CREATE TABLE IF NOT EXISTS __schema_metadata ( + component_type TEXT NOT NULL, + component_name TEXT NOT NULL, + component_key TEXT PRIMARY KEY, + definition JSONB NOT NULL, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )` + ])] + })); + } + + const base = schemaComponent('sc:dumbo:feature:system', { + components: tables, + }); + + const component = { + ...base, + __featureMarker: true as const, + systemTables: { + migrationTracking: options?.migrationTracking !== false, + pongoMetadata: options?.pongoMetadata !== false, + schemaIntrospection: options?.schemaIntrospection !== false, + }, + internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), + } satisfies SystemSchemaComponent; + + // If schemaName provided, wrap in schema component + if ('schemaName' in options && options.schemaName) { + return dumboDatabaseSchema(options.schemaName, {}).addComponent(component); + } + + return component; +} + +export const isSystemSchemaComponent = ( + comp: AnySchemaComponent +): comp is SystemSchemaComponent => { + return comp.schemaComponentKey === 'sc:dumbo:feature:system'; +}; +``` + +### 5. Function Schema Component + +First-class support for stored procedures/functions. + +```typescript +export type FunctionURNType = 'sc:dumbo:function'; +export type FunctionURN = `${FunctionURNType}:${string}`; + +export type FunctionSchemaComponent = SchemaComponent< + FunctionURN, + Readonly<{ + functionName: string; + language?: 'plpgsql' | 'sql' | 'javascript'; // Database-specific + }> +>; + +export const dumboFunction = ( + name: string, + options: SchemaComponentOptions & { + language?: 'plpgsql' | 'sql' | 'javascript'; + } +): FunctionSchemaComponent => { + const base = schemaComponent(`sc:dumbo:function:${name}` as FunctionURN, { + migrations: options.migrations ?? [], + components: options.components ?? [], + }); + + return { + ...base, + functionName: name, + language: options.language, + }; +}; +``` + +## Usage Examples + +### Example 1: Simple Single-Schema Application + +```typescript +// Everything in one schema, system tables auto-added +const simpleApp = dumboDatabase('my_app', { + public: dumboDatabaseSchema('public', { + users: dumboTable('users', { + columns: { + id: dumboColumn('id', { type: 'serial', primaryKey: true }), + email: dumboColumn('email', { type: 'varchar', length: 255 }), + } + }), + posts: dumboTable('posts', { + columns: { + id: dumboColumn('id', { type: 'serial', primaryKey: true }), + userId: dumboColumn('user_id', { type: 'int' }), + } + }), + }) +}); + +// System tables (__migrations, __pongo_collections, __schema_metadata) +// are automatically added to the 'public' schema +``` + +### Example 2: Event Store with Read Models + +```typescript +// Event store and read models in same schema +const hotelApp = dumboDatabase('hotel', { + public: dumboDatabaseSchema('public', { + // Event store feature (opaque - user doesn't see internal tables) + eventStore: eventStoreSchema(), + + // User-defined tables + guests: dumboTable('guests', { + columns: { + id: dumboColumn('id', { type: 'serial', primaryKey: true }), + name: dumboColumn('name', { type: 'varchar', length: 255 }), + } + }), + + reservations: dumboTable('reservations', { + columns: { + id: dumboColumn('id', { type: 'serial', primaryKey: true }), + guestId: dumboColumn('guest_id', { type: 'int' }), + } + }), + }) +}); + +// Generated migrations include: +// - System tables: __migrations, __pongo_collections, __schema_metadata +// - Event store tables: streams, messages, subscriptions +// - Event store function: emt_append_to_stream +// - User tables: guests, reservations +``` + +### Example 3: Separated Schemas + +```typescript +// Event store and read models in separate schemas +const hotelApp = dumboDatabase('hotel', { + // Dedicated system schema + system: systemSchema('system'), + + // Event store in its own schema + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + + // Read models in separate schema + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), + }), +}); + +// PostgreSQL generates: +// CREATE SCHEMA system; +// CREATE TABLE system.__migrations (...); +// CREATE SCHEMA event_store; +// CREATE TABLE event_store.streams (...); +// CREATE SCHEMA read_models; +// CREATE TABLE read_models.guests (...); +``` + +### Example 4: Event Store with Inline Projections + +```typescript +const hotelApp = dumboDatabase('hotel', { + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema({ + // Inline projections become tables in same schema + inlineProjections: { + guestSummary: pongoSchema.collection('guest_summary'), + reservationSummary: pongoSchema.collection('reservation_summary'), + } + }), + }), + + read_models: dumboDatabaseSchema('read_models', { + // Or define Pongo collections as a feature + collections: pongoCollectionsSchema({ + guests: pongoSchema.collection('guests'), + reservations: pongoSchema.collection('reservations'), + }), + }), +}); + +// event_store schema contains: +// - streams, messages, subscriptions (event store core) +// - guest_summary, reservation_summary (inline projections) +// +// read_models schema contains: +// - guests, reservations (Pongo collections → Dumbo tables) +``` + +### Example 5: Multiple Databases with Database Group + +```typescript +// Database group for organizing related databases +const hotelSystemGroup = databaseGroup('hotel_system', { + // Operational database + operational: dumboDatabase('hotel_operational', { + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), + }), + }), + + // Analytics database + analytics: dumboDatabase('hotel_analytics', { + public: dumboDatabaseSchema('public', { + events: dumboTable('events', { /* ... */ }), + metrics: dumboTable('metrics', { /* ... */ }), + }), + }), + + // Reporting database + reporting: dumboDatabase('hotel_reporting', { + public: dumboDatabaseSchema('public', { + reports: dumboTable('reports', { /* ... */ }), + }), + }), +}, { + // Shared system schema across all databases + shared: { + systemSchema: systemSchema('shared_system'), + } +}); +``` + +## System Schema Placement Strategy + +### Single Schema +When database has exactly one schema, system tables are automatically added to that schema. + +```typescript +const db = dumboDatabase('app', { + public: dumboDatabaseSchema('public', { /* ... */ }) +}); +// ✅ System tables → 'public' schema +``` + +### Multiple Schemas Without Explicit System Schema +Default behavior: Use database type's default schema convention. + +```typescript +const db = dumboDatabase('app', { + event_store: dumboDatabaseSchema('event_store', { /* ... */ }), + read_models: dumboDatabaseSchema('read_models', { /* ... */ }), +}); + +// PostgreSQL: Check for 'public' schema, otherwise use first schema +// ✅ If 'public' exists → system tables go there +// ✅ Otherwise → system tables go to 'event_store' (first schema) + +// SQLite: Use first schema (schema names become table prefixes) +// ✅ System tables → 'event_store' schema (prefixed: event_store___migrations) +``` + +### Explicit System Schema +User can explicitly place system schema anywhere: + +```typescript +// Option A: Dedicated system schema +const db = dumboDatabase('app', { + system: systemSchema('admin'), // Returns DatabaseSchemaSchemaComponent + event_store: dumboDatabaseSchema('event_store', { /* ... */ }), + read_models: dumboDatabaseSchema('read_models', { /* ... */ }), +}); + +// Option B: System schema within existing schema +const db = dumboDatabase('app', { + event_store: dumboDatabaseSchema('event_store', { + system: systemSchema(), // Just the component + eventStore: eventStoreSchema(), + }), + read_models: dumboDatabaseSchema('read_models', { /* ... */ }), +}); +``` + +## Generic Sharding Strategy + +### Sharding Configuration + +```typescript +export type ShardingStrategy = { + // How to identify this shard dimension + shardKey: TShardKey; + + // What values this shard can have + shardValues: readonly string[] | 'dynamic'; + + // Naming pattern for the sharded resource + namingPattern: string | ((shard: string, original: string) => string); + + // Which resources should NOT be sharded (shared across all shards) + exclude?: string[] | ((resourceName: string) => boolean); +}; + +export type ShardingLevel = 'table' | 'schema' | 'database' | 'database-group'; + +export type ShardingConfig = { + level: ShardingLevel; + strategy: ShardingStrategy; + + // Optional: Apply sharding only to specific components + applyTo?: string[]; +}; +``` + +### Predefined Sharding Strategies + +```typescript +export const shardingStrategies = { + // Multi-tenancy: Shard by tenant ID + multiTenant: (tenants: string[] | 'dynamic'): ShardingStrategy => ({ + shardKey: 'tenant_id', + shardValues: tenants, + namingPattern: '{shard}_{resource}', + exclude: ['system'], // Don't shard system schema + }), + + // Geographic regions + region: (regions: string[]): ShardingStrategy => ({ + shardKey: 'region', + shardValues: regions, + namingPattern: '{resource}_{shard}', + }), + + // Time-based partitioning + timePartition: (periods: string[]): ShardingStrategy => ({ + shardKey: 'period', + shardValues: periods, // e.g., ['2024_01', '2024_02', ...] + namingPattern: '{resource}_{shard}', + }), + + // Customer-based sharding + customerId: (customerIds: string[] | 'dynamic'): ShardingStrategy => ({ + shardKey: 'customer_id', + shardValues: customerIds, + namingPattern: 'cust_{shard}_{resource}', + }), +}; +``` + +### Sharding Examples + +#### Schema-Level Sharding (Multi-Tenancy) + +```typescript +const hotelApp = dumboDatabase('hotel', { + system: systemSchema('system'), // Not sharded + + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), + }), +}, { + sharding: { + level: 'schema', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott', 'hyatt']), + } +}); + +// PostgreSQL generates: +// CREATE SCHEMA system; -- Not sharded +// CREATE SCHEMA hilton_event_store; +// CREATE TABLE hilton_event_store.streams (...); +// CREATE SCHEMA hilton_read_models; +// CREATE TABLE hilton_read_models.guests (...); +// CREATE SCHEMA marriott_event_store; +// CREATE TABLE marriott_event_store.streams (...); +// CREATE SCHEMA marriott_read_models; +// CREATE TABLE marriott_read_models.guests (...); +// ... +``` + +#### Database-Level Sharding + +```typescript +const hotelApp = dumboDatabase('hotel', { + event_store: dumboDatabaseSchema('event_store', { /* ... */ }), + read_models: dumboDatabaseSchema('read_models', { /* ... */ }), +}, { + sharding: { + level: 'database', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + } +}); + +// PostgreSQL generates separate databases: +// CREATE DATABASE hilton_hotel; +// CREATE DATABASE marriott_hotel; + +// SQLite generates separate files: +// hilton_hotel.db +// marriott_hotel.db +``` + +#### Table-Level Sharding (Time Partitioning) + +```typescript +const analyticsDb = dumboDatabase('analytics', { + public: dumboDatabaseSchema('public', { + events: dumboTable('events', { /* ... */ }), + metrics: dumboTable('metrics', { /* ... */ }), + }), +}, { + sharding: { + level: 'table', + strategy: shardingStrategies.timePartition(['2024_01', '2024_02', '2024_03']), + applyTo: ['events'], // Only shard events table + } +}); + +// Generates: +// CREATE TABLE events_2024_01 (...); +// CREATE TABLE events_2024_02 (...); +// CREATE TABLE events_2024_03 (...); +// CREATE TABLE metrics (...); -- Not sharded +``` + +#### Database Group Sharding + +```typescript +const hotelSystemGroup = databaseGroup('hotel_system', { + operational: dumboDatabase('hotel_operational', { /* ... */ }), + analytics: dumboDatabase('hotel_analytics', { /* ... */ }), + reporting: dumboDatabase('hotel_reporting', { /* ... */ }), +}, { + sharding: { + level: 'database', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + }, + shared: { + systemSchema: systemSchema('shared_system'), + } +}); + +// Generates for each tenant: +// Tenant 'hilton': +// Database: hilton_hotel_operational +// Database: hilton_hotel_analytics +// Database: hilton_hotel_reporting +// Shared: shared_system schema (referenced from all databases) +// +// Tenant 'marriott': +// Database: marriott_hotel_operational +// Database: marriott_hotel_analytics +// Database: marriott_hotel_reporting +// Shared: shared_system schema (referenced from all databases) +``` + +### Component-Level Sharding Override + +```typescript +// Different sharding strategies for different schemas +const hotelApp = dumboDatabase('hotel', { + system: systemSchema('system'), // No sharding + + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }, { + // Override: shard event store by tenant + sharding: { + level: 'schema', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + } + }), + + analytics: dumboDatabaseSchema('analytics', { + events: dumboTable('events', { /* ... */ }), + }, { + // Override: shard analytics by region + sharding: { + level: 'table', + strategy: shardingStrategies.region(['us_east', 'us_west', 'eu']), + } + }), +}); + +// Generates: +// CREATE SCHEMA system; +// CREATE SCHEMA hilton_event_store; +// CREATE SCHEMA marriott_event_store; +// CREATE SCHEMA analytics; +// CREATE TABLE analytics.events_us_east (...); +// CREATE TABLE analytics.events_us_west (...); +// CREATE TABLE analytics.events_eu (...); +``` + +## Database Group Design + +### Database Group Component + +```typescript +export type DatabaseGroup< + Databases extends Record = Record +> = { + groupName: string; + databases: Databases; + + // Group-level sharding configuration + sharding?: ShardingConfig; + + // Resources shared across all databases in group + shared?: { + systemSchema?: SystemSchemaComponent; + }; +}; + +export const databaseGroup = >( + groupName: string, + databases: T, + options?: { + sharding?: ShardingConfig; + shared?: { + systemSchema?: SystemSchemaComponent; + }; + } +): DatabaseGroup => ({ + groupName, + databases, + sharding: options?.sharding, + shared: options?.shared, +}); +``` + +### Use Cases for Database Groups + +1. **Logical Organization**: Group related databases for documentation and architecture diagrams +2. **Shared Sharding**: Apply same sharding strategy across multiple databases +3. **Shared Resources**: Single system schema referenced by multiple databases +4. **CQRS/Event Sourcing**: Separate databases for commands, queries, and events +5. **Polyglot Persistence**: Different databases for different concerns (operational, analytics, reporting) + +## Migration Generation + +### Database-Agnostic Schema, Database-Specific Migrations + +Schema definitions are portable; migration generation considers database type: + +```typescript +// Schema definition (database-agnostic) +const hotelSchema = dumboDatabase('hotel', { + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + }), +}); + +// PostgreSQL migration generation +const pgMigrations = generateMigrations(hotelSchema, { + databaseType: 'postgresql' +}); +// Generates: +// CREATE SCHEMA event_store; +// CREATE TABLE event_store.streams (...); +// CREATE SCHEMA read_models; +// CREATE TABLE read_models.guests (...); + +// SQLite migration generation +const sqliteMigrations = generateMigrations(hotelSchema, { + databaseType: 'sqlite', + sqliteStrategy: 'prefix-tables' // or 'separate-files' +}); +// prefix-tables generates: +// CREATE TABLE event_store_streams (...); +// CREATE TABLE read_models_guests (...); +// +// separate-files generates: +// File: event_store.db → CREATE TABLE streams (...); +// File: read_models.db → CREATE TABLE guests (...); +``` + +### Migration Collection from Features + +The migration orchestrator detects feature components and extracts their internal migrations: + +```typescript +export const collectAllMigrations = ( + database: DatabaseSchemaComponent, +): SQLMigration[] => { + const migrations: SQLMigration[] = []; + + for (const schema of database.schemas.values()) { + for (const component of schema.components.values()) { + if (isFeatureSchemaComponent(component)) { + // Feature component: extract internal components + for (const internalComp of component.internalComponents.values()) { + if (isTableComponent(internalComp)) { + migrations.push(...internalComp.migrations); + } else if (isFunctionComponent(internalComp)) { + migrations.push(...internalComp.migrations); + } + } + } else if (isTableComponent(component)) { + // Regular table + migrations.push(...component.migrations); + } else if (isFunctionComponent(component)) { + // Regular function + migrations.push(...component.migrations); + } + } + } + + return migrations; +}; +``` + +## Type Generation + +### Exposing Feature Internals in Generated Types + +While features are opaque at definition time, generated types expose internal structure for type-safe queries: + +```typescript +// Schema definition (event store is opaque) +const hotelSchema = dumboDatabase('hotel', { + public: dumboDatabaseSchema('public', { + eventStore: eventStoreSchema(), + guests: dumboTable('guests', { /* ... */ }), + }) +}); + +// Generated types expose all tables (including event store internals) +export type HotelSchema = { + public: { + // Event store tables (exposed for type-safe queries) + streams: { + stream_id: string; + stream_position: number; + partition: string; + stream_type: string; + stream_metadata: unknown; + is_archived: boolean; + }; + messages: { + stream_id: string; + stream_position: number; + partition: string; + message_kind: string; + message_data: unknown; + message_metadata: unknown; + message_schema_version: string; + message_type: string; + message_id: string; + is_archived: boolean; + global_position: number; + created: Date; + }; + subscriptions: { + subscription_id: string; + version: number; + partition: string; + last_processed_position: number; + }; + + // User-defined tables + guests: { + id: number; + name: string; + email: string; + }; + }; +}; + +// Usage with type-safe queries +const stream = await db + .from('streams') // TypeScript knows this exists + .where('stream_id', '=', 'guest-123') + .select(['stream_position', 'stream_type']); +// Type: { stream_position: number, stream_type: string }[] +``` + +## Integration with Emmett Architecture + +Feature components are separate from Emmett architectural components but can be mapped: + +```typescript +// Emmett: Logical architecture (business components with ports) +const hotelManagementContainer = emmettArch.container('hotel-management', { + eventStore: emmettArch.component('event-store', { + ports: { + exposes: { + commands: { + appendToStream: (/* ... */) => Promise.resolve(), + }, + }, + }, + }), + + guests: emmettArch.component('guests', { + ports: { + exposes: { + queries: { + getGuestByExternalId: query(), + }, + }, + }, + }), + + reservations: emmettArch.component('reservations', { + ports: { + requires: { + guests: { + getGuestByExternalId: query(), + }, + }, + }, + }), +}); + +// Dumbo: Physical schema (database structure) +const hotelSchema = dumboDatabase('hotel', { + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { /* ... */ }), + reservations: dumboTable('reservations', { /* ... */ }), + }), +}); + +// Mapping layer (optional - for documentation/tooling) +const deployment = { + architecture: hotelManagementContainer, + schema: hotelSchema, + mapping: { + // Map Emmett components to Dumbo schemas/tables + 'event-store': 'event_store.eventStore', + 'guests': 'read_models.guests', + 'reservations': 'read_models.reservations', + } +}; +``` + +## Design Decisions + +### 1. Why Feature Components Extend SchemaComponent? +- **Uniform Composition**: Features compose like tables, indexes, etc. +- **Consistent API**: Same `addComponent()` pattern everywhere +- **Type Safety**: Generic `SchemaComponent` infrastructure works for features +- **Migration System**: Features participate in migration collection automatically + +### 2. Why System Schema is a Feature Component? +- **Consistent Placement**: Same composition rules as other features +- **Flexible Location**: Can be in dedicated schema or mixed with application schemas +- **Opaque Internals**: Users don't need to know about __migrations, __pongo_collections tables +- **Customizable**: Power users can configure which system tables to include + +### 3. Why Generic Sharding vs. Hardcoded Multi-Tenancy? +- **Extensibility**: Supports time partitioning, regional sharding, custom strategies +- **Composition**: Can combine multiple sharding dimensions in the future +- **Flexibility**: Same mechanism for different use cases (multi-tenancy, scaling, compliance) +- **Simplicity**: Single concept to learn instead of multiple special cases + +### 4. Why Database Groups? +- **Organization**: Large systems have multiple related databases +- **Shared Resources**: System schema can be shared across databases +- **Consistent Sharding**: Apply same tenant strategy to command/query/event databases +- **Documentation**: Architecture diagrams show logical database groupings + +### 5. Why Portable Schema Definitions? +- **Developer Experience**: Write once, deploy to PostgreSQL or SQLite +- **Testing**: Test with SQLite, deploy to PostgreSQL +- **Flexibility**: Change database type without rewriting schema +- **Separation of Concerns**: Schema = logical structure, migrations = physical implementation + +## Future Enhancements + +### 1. Composable Sharding (Out of Scope for Initial Implementation) + +Support multiple sharding dimensions: + +```typescript +{ + sharding: [ + { level: 'database', strategy: shardingStrategies.multiTenant(['hilton']) }, + { level: 'schema', strategy: shardingStrategies.region(['us', 'eu']) }, + { level: 'table', strategy: shardingStrategies.timePartition(['2024_01']) }, + ] +} +// Generates: Database hilton_hotel → Schema us_event_store → Table events_2024_01 +``` + +### 2. Pattern-Based Exclusion (Future) + +Support patterns in sharding exclusion: + +```typescript +{ + sharding: { + strategy: tenantSharding, + exclude: [ + 'system', // Exact match + '__*', // Glob pattern: all system tables + /^temp_/, // Regex: temporary tables + (name) => name.startsWith('cache_'), // Function + ] + } +} +``` + +### 3. Dynamic Shard Provisioning (Out of Scope for Schema Definition) + +Runtime provisioning of new shards (tenants, regions, etc.): + +```typescript +// This is a runtime concern, not schema definition +const provisioner = createShardProvisioner(schema, config); +await provisioner.provisionShard(connection, 'new_tenant_id'); +``` + +### 4. Cross-Database Relationships (Future) + +Support foreign keys across sharded databases: + +```typescript +// Define relationship that spans databases +const relationship = crossDatabaseForeignKey( + 'read_models.reservations.guest_id', + 'read_models.guests.id', + { onDelete: 'CASCADE' } +); +``` + +### 5. Migration Dependency Graph (Future) + +Explicit migration dependencies for complex scenarios: + +```typescript +const createUsersMigration = sqlMigration('create_users', [/* ... */]); +const createPostsMigration = sqlMigration('create_posts', [/* ... */], { + dependsOn: [createUsersMigration], // Posts table needs users table first +}); +``` + +## Open Questions + +### Q1: Function Component Placement +Should functions be: +- Top-level components in schema (like tables)? +- Nested within table components? +- Only within feature components? + +**Current decision**: Top-level and within features (flexible) + +### Q2: SQLite Multiple Schema Strategy Default +For SQLite with multiple schemas, default to: +- Table prefixing (simpler, single file) +- Separate database files (stronger isolation) +- Error (force explicit choice) + +**Current decision**: Table prefixing (simpler default, user can override) + +### Q3: System Schema Naming Convention +Should system schema default name be: +- `__dumbo_system` (clear it's framework) +- `_system` (shorter) +- `system` (clean but might conflict) + +**Current decision**: `system` for dedicated schema, auto-add to first schema otherwise + +### Q4: Sharding Naming Pattern Syntax +Support only string templates, or also functions? +- String: `'{shard}_{resource}'` +- Function: `(shard, resource) => ...` + +**Current decision**: Both (string for simplicity, function for flexibility) + +### Q5: Feature Component Registration +Should there be a registry for custom feature components? + +```typescript +// Register custom feature +registerFeatureComponent('custom_feature', { + detect: (comp) => comp.schemaComponentKey.startsWith('sc:custom:'), + extractMigrations: (comp) => { /* ... */ }, +}); +``` + +**Current decision**: Not yet - keep simple, add if needed + +## Implementation Checklist + +### Phase 1: Core Feature Components +- [ ] `FeatureSchemaComponent` base type +- [ ] `isFeatureSchemaComponent()` type guard +- [ ] `EventStoreSchemaComponent` implementation +- [ ] `PongoCollectionsSchemaComponent` implementation +- [ ] `SystemSchemaComponent` implementation +- [ ] `FunctionSchemaComponent` implementation + +### Phase 2: System Schema Placement +- [ ] Auto-detection logic for single schema +- [ ] Database type default schema logic (PostgreSQL 'public', etc.) +- [ ] Explicit system schema placement +- [ ] System schema component tests + +### Phase 3: Generic Sharding +- [ ] `ShardingStrategy` type definition +- [ ] `ShardingConfig` type definition +- [ ] Predefined strategies (`multiTenant`, `region`, `timePartition`) +- [ ] Sharding at database level +- [ ] Sharding at schema level +- [ ] Sharding at table level +- [ ] Exclusion logic for shared resources + +### Phase 4: Database Groups +- [ ] `DatabaseGroup` type definition +- [ ] `databaseGroup()` factory function +- [ ] Shared system schema across databases +- [ ] Group-level sharding +- [ ] Migration generation for groups + +### Phase 5: Migration Generation +- [ ] Detect feature components in migration collector +- [ ] Extract internal migrations from features +- [ ] Database-specific migration generation (PostgreSQL vs SQLite) +- [ ] SQLite table prefixing strategy +- [ ] SQLite separate files strategy +- [ ] Sharding-aware migration generation + +### Phase 6: Type Generation +- [ ] Extract internal structure from features +- [ ] Generate types for event store tables +- [ ] Generate types for Pongo collections +- [ ] Generate types for custom features +- [ ] Exclude system tables from main types (optional) + +### Phase 7: Integration & Documentation +- [ ] Dumbo schema API updates +- [ ] Pongo schema integration +- [ ] Emmett architecture mapping examples +- [ ] Unit tests for all components +- [ ] Integration tests with PostgreSQL +- [ ] Integration tests with SQLite +- [ ] Documentation and examples + +## Related Files + +- [MIGRATION_UNIFICATION_PLAN.md](./MIGRATION_UNIFICATION_PLAN.md) - Migration system design +- [schemaComponent.ts](./schemaComponent.ts) - Base schema component implementation +- [dumboSchema.ts](./dumboSchema/dumboSchema.ts) - Dumbo schema builder API +- [pongo/schema/index.ts](../../../pongo/src/core/schema/index.ts) - Pongo schema system +- Emmett Architecture (external package) - Component/container/system definitions + +## References + +- [Dumbo Migration Unification Plan](./MIGRATION_UNIFICATION_PLAN.md) - Context on migration system +- [Pongo Strongly Typed Client](https://event-driven.io/en/pongo_strongly_typed_client/) - Pongo collection schema +- [Emmett Projections Testing](https://event-driven.io/en/emmett_projections_testing/) - Event store and projections +- C4 Model - Architectural component hierarchy (system → container → component) + +--- + +**Document Version**: 1.0 +**Last Updated**: 2025-10-31 +**Status**: Design Proposal +**Authors**: Based on collaborative design discussion From 560748ba4f6a21ce82a6ace5dc5f40ac569bc2a2 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 13 Nov 2025 10:21:29 +0100 Subject: [PATCH 20/39] Refactored column schema component to take all SQLToken parameters like column type etc. --- .../core/schema/MIGRATION_UNIFICATION_PLAN.md | 440 +++++++++----- .../components/columnSchemaComponent.ts | 4 +- .../schema/components/indexSchemaComponent.ts | 32 +- .../schema/components/tableSchemaComponent.ts | 20 +- .../core/schema/dumboSchema/dumboSchema.ts | 3 +- .../dumboSchema/dumboSchema.unit.spec.ts | 136 +++-- .../dumbo/src/core/schema/schema_feature.md | 536 +++++++++++------- 7 files changed, 731 insertions(+), 440 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md index c867b058..9299d4c5 100644 --- a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md +++ b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md @@ -1,11 +1,13 @@ # Schema Component Migration Unification Plan ## Overview + This plan details the unification of SQL migrations and TypeScript schema definitions in the Dumbo schema component system. The goal is to support both SQL-first and TypeScript-first workflows while maintaining a clear source of truth and enabling strongly-typed query builders for Pongo. ## Context and Current State ### Key Files + - **Core Schema Component**: [src/packages/dumbo/src/core/schema/schemaComponent.ts](src/packages/dumbo/src/core/schema/schemaComponent.ts) - Lines 3-146 - **Component Implementations**: - [src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts) @@ -22,7 +24,9 @@ This plan details the unification of SQL migrations and TypeScript schema defini - **Pongo Integration Target**: [src/packages/pongo/src/core/schema/index.ts](src/packages/pongo/src/core/schema/index.ts) ### Current Problem + The system cannot distinguish between: + 1. **Initial schema definitions** (CREATE TABLE) 2. **Schema evolution** (ALTER TABLE) 3. **Data migrations** (UPDATE/INSERT) @@ -34,6 +38,7 @@ All migrations are stored in a single `migrations` array and bubble up through t ### Core Principle: TypeScript as Source of Truth (When Present) **The Rule**: When TypeScript definitions exist, they represent the desired schema state. Migrations are either: + 1. **Generated** from TypeScript definitions (if no migrations provided) 2. **Validated** against TypeScript definitions (if migrations provided) 3. **Trusted** completely (if no TypeScript definitions provided - SQL-first mode) @@ -41,39 +46,38 @@ All migrations are stored in a single `migrations` array and bubble up through t ### Three Usage Modes #### 1. TypeScript-First (Generates Migrations) + ```typescript const table = tableSchemaComponent({ tableName: 'users', columns: { id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }) - } + email: column('email', { type: 'varchar', length: 255 }), + }, // No migrations = will generate CREATE TABLE IF NOT EXISTS }); ``` #### 2. SQL-First (Migrations as Source of Truth) + ```typescript const table = tableSchemaComponent({ tableName: 'users', - migrations: [ - sqlMigration('001_create', [SQL`CREATE TABLE users ...`]) - ] + migrations: [sqlMigration('001_create', [SQL`CREATE TABLE users ...`])], // No columns = trust the SQL completely }); ``` #### 3. Hybrid with Validation + ```typescript const table = tableSchemaComponent({ tableName: 'users', columns: { id: column('id', { type: 'serial' }), - email: column('email', { type: 'varchar' }) + email: column('email', { type: 'varchar' }), }, - migrations: [ - sqlMigration('001_create', [SQL`CREATE TABLE users ...`]) - ] + migrations: [sqlMigration('001_create', [SQL`CREATE TABLE users ...`])], // Both provided = TypeScript is truth, validate migrations match }); ``` @@ -86,7 +90,17 @@ const table = tableSchemaComponent({ ```typescript export type ColumnDefinition = { - type: 'serial' | 'varchar' | 'int' | 'bigint' | 'text' | 'timestamp' | 'boolean' | 'uuid' | 'json' | 'jsonb'; + type: + | 'serial' + | 'varchar' + | 'int' + | 'bigint' + | 'text' + | 'timestamp' + | 'boolean' + | 'uuid' + | 'json' + | 'jsonb'; nullable?: boolean; primaryKey?: boolean; unique?: boolean; @@ -106,8 +120,8 @@ export type ColumnSchemaComponent = SchemaComponent< ColumnURN, Readonly<{ columnName: string; - definition?: ColumnDefinition; // NEW: Optional definition for TypeScript-first - generateMigration?: () => SQL; // NEW: Self-contained migration generation + definition?: ColumnDefinition; // NEW: Optional definition for TypeScript-first + generateMigration?: () => SQL; // NEW: Self-contained migration generation }> >; @@ -115,7 +129,8 @@ export type ColumnSchemaComponent = SchemaComponent< export const generateColumnSQL = (column: ColumnSchemaComponent): string => { if (!column.definition) return `${column.columnName} TEXT`; // fallback - const { type, nullable, primaryKey, unique, length, defaultValue } = column.definition; + const { type, nullable, primaryKey, unique, length, defaultValue } = + column.definition; let sql = `${column.columnName} ${type.toUpperCase()}`; if (length) sql += `(${length})`; @@ -131,6 +146,7 @@ export const generateColumnSQL = (column: ColumnSchemaComponent): string => { ### Phase 2: Nested Schema Snapshot System **Complete directory structure**: + ``` .dumbo/ snapshots/ # Current state + migration tracking @@ -191,12 +207,12 @@ export const usersSnapshot = { tableName: 'users', columns: { id: { type: 'serial' as const, primaryKey: true }, - email: { type: 'varchar' as const, length: 255, nullable: false } + email: { type: 'varchar' as const, length: 255, nullable: false }, }, indexes: { - idx_users_email: { columns: ['email'], unique: false } + idx_users_email: { columns: ['email'], unique: false }, }, - migrations: [] // No migrations provided, will be generated + migrations: [], // No migrations provided, will be generated } as const; // SCENARIO 2: SQL-first (only migrations provided) @@ -215,13 +231,19 @@ export const ordersSnapshot = { // Introspected from database after applying migrations id: { type: 'serial' as const, primaryKey: false }, total: { type: 'decimal' as const, precision: 10, scale: 2 }, - user_id: { type: 'int' as const, nullable: true } + user_id: { type: 'int' as const, nullable: true }, }, indexes: {}, migrations: [ - { name: '001_create', sqls: ['CREATE TABLE orders (id SERIAL, total DECIMAL(10,2))'] }, - { name: '002_add_user', sqls: ['ALTER TABLE orders ADD COLUMN user_id INT'] } - ] + { + name: '001_create', + sqls: ['CREATE TABLE orders (id SERIAL, total DECIMAL(10,2))'], + }, + { + name: '002_add_user', + sqls: ['ALTER TABLE orders ADD COLUMN user_id INT'], + }, + ], } as const; // SCENARIO 3: Hybrid (TypeScript definitions + SQL migrations) @@ -237,12 +259,17 @@ export const postsSnapshot = { columns: { // TypeScript definitions as source of truth id: { type: 'serial' as const, primaryKey: true }, - title: { type: 'varchar' as const, length: 255, nullable: false } + title: { type: 'varchar' as const, length: 255, nullable: false }, }, indexes: {}, migrations: [ - { name: '001_custom', sqls: ['CREATE TABLE posts (id SERIAL PRIMARY KEY, title VARCHAR(255), internal_field JSONB)'] } - ] + { + name: '001_custom', + sqls: [ + 'CREATE TABLE posts (id SERIAL PRIMARY KEY, title VARCHAR(255), internal_field JSONB)', + ], + }, + ], } as const; // .dumbo/snapshots/schemas/public.snapshot.ts @@ -255,8 +282,8 @@ export const publicSnapshot = { tables: { users: usersSnapshot, posts: postsSnapshot, - comments: commentsSnapshot - } + comments: commentsSnapshot, + }, } as const; // .dumbo/snapshots/myapp.snapshot.ts @@ -269,12 +296,13 @@ export const databaseSnapshot = { timestamp: '2024-01-15T10:30:00Z', schemas: { public: publicSnapshot, - analytics: analyticsSnapshot - } + analytics: analyticsSnapshot, + }, } as const; ``` **Benefits of nested structure**: + - Minimizes git merge conflicts (each table in separate file) - Clear ownership and change tracking - Natural composition mirroring schema hierarchy @@ -310,13 +338,13 @@ export type TableSnapshot = { tableName: string; columns: Record; indexes: Record; - migrations: SQLMigration[]; // Track migrations defined in component + migrations: SQLMigration[]; // Track migrations defined in component }; export type SchemaSnapshot = { schemaName: string; tables: Record; - migrations: SQLMigration[]; // Schema-level migrations + migrations: SQLMigration[]; // Schema-level migrations }; export type DatabaseSnapshot = { @@ -324,7 +352,7 @@ export type DatabaseSnapshot = { version: string; timestamp: string; schemas: Record; - migrations: SQLMigration[]; // Database-level migrations + migrations: SQLMigration[]; // Database-level migrations }; ``` @@ -340,7 +368,7 @@ export type DatabaseSnapshot = { // Add to tableSchemaComponent.ts export const generateTableMigrations = ( table: TableSchemaComponent, - snapshot?: TableSnapshot + snapshot?: TableSnapshot, ): SQLMigration[] => { // If migrations already provided, return them (SQL-first mode) if (table.migrations.length > 0) { @@ -357,15 +385,15 @@ export const generateTableMigrations = ( if (!snapshot) { // No snapshot = first time creation const columnDefinitions = Array.from(table.columns.values()) - .map(col => generateColumnSQL(col)) + .map((col) => generateColumnSQL(col)) .join(',\n '); migrations.push( sqlMigration(`create_table_${table.tableName}`, [ SQL`CREATE TABLE IF NOT EXISTS ${table.tableName} ( ${columnDefinitions} -)` - ]) +)`, + ]), ); } else { // Generate ALTER statements based on diff with snapshot @@ -374,8 +402,8 @@ export const generateTableMigrations = ( for (const newColumn of changes.addedColumns) { migrations.push( sqlMigration(`add_${newColumn.columnName}_to_${table.tableName}`, [ - SQL`ALTER TABLE ${table.tableName} ADD COLUMN IF NOT EXISTS ${generateColumnSQL(newColumn)}` - ]) + SQL`ALTER TABLE ${table.tableName} ADD COLUMN IF NOT EXISTS ${generateColumnSQL(newColumn)}`, + ]), ); } } @@ -391,13 +419,15 @@ export const generateTableMigrations = ( }; // Table component enhanced constructor -export const tableSchemaComponent = (options: TableOptions): TableSchemaComponent => { +export const tableSchemaComponent = ( + options: TableOptions, +): TableSchemaComponent => { // ... existing implementation ... const component = { // ... existing properties ... generateMigrations: (snapshot?: TableSnapshot) => - generateTableMigrations(component, snapshot) + generateTableMigrations(component, snapshot), }; return component; @@ -411,14 +441,14 @@ export const tableSchemaComponent = (options: TableOptions): TableSchemaComponen ```typescript export const generateIndexMigration = ( tableName: string, - index: IndexSchemaComponent + index: IndexSchemaComponent, ): SQLMigration => { const uniqueClause = index.unique ? 'UNIQUE ' : ''; const columns = index.columns.join(', '); return sqlMigration(`create_index_${index.indexName}`, [ SQL`CREATE ${uniqueClause}INDEX IF NOT EXISTS ${index.indexName} - ON ${tableName}(${columns})` + ON ${tableName}(${columns})`, ]); }; ``` @@ -430,7 +460,7 @@ export const generateIndexMigration = ( ```typescript export const generateSchemaMigrations = ( schema: DatabaseSchemaSchemaComponent, - snapshot?: SchemaSnapshot + snapshot?: SchemaSnapshot, ): SQLMigration[] => { if (schema.migrations.length > 0) { return schema.migrations; @@ -441,14 +471,16 @@ export const generateSchemaMigrations = ( // Create schema if doesn't exist migrations.push( sqlMigration(`create_schema_${schema.schemaName}`, [ - SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}` - ]) + SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}`, + ]), ); // Tables will generate their own migrations // But we collect them here for ordering for (const table of schema.tables.values()) { - migrations.push(...table.generateMigrations(snapshot?.tables[table.tableName])); + migrations.push( + ...table.generateMigrations(snapshot?.tables[table.tableName]), + ); } return migrations; @@ -464,7 +496,7 @@ The database component is the root and orchestrates migration generation by trav ```typescript export const collectAllMigrations = ( database: DatabaseSchemaComponent, - snapshot?: DatabaseSnapshot + snapshot?: DatabaseSnapshot, ): SQLMigration[] => { const migrations: SQLMigration[] = []; @@ -485,8 +517,8 @@ export const collectAllMigrations = ( // 1. Database-level setup migrations.push( sqlMigration(`setup_database_${database.databaseName}`, [ - SQL`-- Database setup for ${database.databaseName}` - ]) + SQL`-- Database setup for ${database.databaseName}`, + ]), ); // 2. Traverse schemas @@ -496,8 +528,8 @@ export const collectAllMigrations = ( // Create schema migrations.push( sqlMigration(`create_schema_${schema.schemaName}`, [ - SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}` - ]) + SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}`, + ]), ); // 3. Collect all tables first (structure only, no foreign keys) @@ -515,7 +547,7 @@ export const collectAllMigrations = ( const tableMigrations = table.generateMigrations(tableSnapshot); // Separate table creation from index creation - tableMigrations.forEach(m => { + tableMigrations.forEach((m) => { if (m.name.includes('index')) { indexCreations.push(m); } else { @@ -526,8 +558,8 @@ export const collectAllMigrations = ( } // Add in correct order - migrations.push(...tableCreations); // All tables first - migrations.push(...indexCreations); // Then all indexes + migrations.push(...tableCreations); // All tables first + migrations.push(...indexCreations); // Then all indexes } // Future: Foreign key constraints would go here @@ -537,13 +569,15 @@ export const collectAllMigrations = ( }; // Enhanced database component -export const databaseSchemaComponent = (options: DatabaseOptions): DatabaseSchemaComponent => { +export const databaseSchemaComponent = ( + options: DatabaseOptions, +): DatabaseSchemaComponent => { // ... existing implementation ... const component = { // ... existing properties ... collectAllMigrations: (snapshot?: DatabaseSnapshot) => - collectAllMigrations(component, snapshot) + collectAllMigrations(component, snapshot), }; return component; @@ -567,12 +601,12 @@ export type TableChanges = { export const diffTableWithSnapshot = ( table: TableSchemaComponent, - snapshot: TableSnapshot + snapshot: TableSnapshot, ): TableChanges => { const changes: TableChanges = { addedColumns: [], removedColumns: [], - modifiedColumns: [] + modifiedColumns: [], }; // Find added columns @@ -596,7 +630,7 @@ export const diffTableWithSnapshot = ( changes.modifiedColumns.push({ name, from: snapshotColumn, - to: column.definition + to: column.definition, }); } } @@ -604,7 +638,10 @@ export const diffTableWithSnapshot = ( return changes; }; -const columnsEqual = (def: ColumnDefinition | undefined, snap: ColumnSnapshot): boolean => { +const columnsEqual = ( + def: ColumnDefinition | undefined, + snap: ColumnSnapshot, +): boolean => { if (!def) return false; return ( def.type === snap.type && @@ -621,7 +658,9 @@ const columnsEqual = (def: ColumnDefinition | undefined, snap: ColumnSnapshot): **New file to create**: `src/packages/dumbo/src/core/schema/generators/typeGenerator.ts` ```typescript -export const generateTypesFromSnapshot = (snapshot: DatabaseSnapshot): string => { +export const generateTypesFromSnapshot = ( + snapshot: DatabaseSnapshot, +): string => { const lines: string[] = []; lines.push('// Auto-generated database types from schema snapshot'); @@ -715,32 +754,40 @@ export type ImmutabilityValidationResult = { export const validateMigrationImmutability = ( currentMigrations: ReadonlyArray, - snapshotMigrations: ReadonlyArray + snapshotMigrations: ReadonlyArray, ): ImmutabilityValidationResult => { // Check that no migrations were removed for (const snapshotMigration of snapshotMigrations) { - const currentMigration = currentMigrations.find(m => m.name === snapshotMigration.name); + const currentMigration = currentMigrations.find( + (m) => m.name === snapshotMigration.name, + ); if (!currentMigration) { return { valid: false, - error: `Migration '${snapshotMigration.name}' was removed from component!\n` + - `Migrations cannot be deleted once defined.\n` + - `If you need to undo a migration, add a new migration that reverses it.` + error: + `Migration '${snapshotMigration.name}' was removed from component!\n` + + `Migrations cannot be deleted once defined.\n` + + `If you need to undo a migration, add a new migration that reverses it.`, }; } // Check that migration content hasn't changed - const snapshotSqls = snapshotMigration.sqls.map(sql => sql.toString()).join('\n'); - const currentSqls = currentMigration.sqls.map(sql => sql.toString()).join('\n'); + const snapshotSqls = snapshotMigration.sqls + .map((sql) => sql.toString()) + .join('\n'); + const currentSqls = currentMigration.sqls + .map((sql) => sql.toString()) + .join('\n'); if (snapshotSqls !== currentSqls) { return { valid: false, - error: `Migration '${snapshotMigration.name}' has been modified!\n` + - `Original SQL:\n${snapshotSqls}\n\n` + - `Current SQL:\n${currentSqls}\n\n` + - `Migrations must be immutable once defined. Add a new migration instead.` + error: + `Migration '${snapshotMigration.name}' has been modified!\n` + + `Original SQL:\n${snapshotSqls}\n\n` + + `Current SQL:\n${currentSqls}\n\n` + + `Migrations must be immutable once defined. Add a new migration instead.`, }; } } @@ -751,10 +798,10 @@ export const validateMigrationImmutability = ( // Helper to detect new migrations export const getNewMigrations = ( currentMigrations: ReadonlyArray, - snapshotMigrations: ReadonlyArray + snapshotMigrations: ReadonlyArray, ): SQLMigration[] => { - const snapshotNames = new Set(snapshotMigrations.map(m => m.name)); - return currentMigrations.filter(m => !snapshotNames.has(m.name)); + const snapshotNames = new Set(snapshotMigrations.map((m) => m.name)); + return currentMigrations.filter((m) => !snapshotNames.has(m.name)); }; ``` @@ -770,7 +817,11 @@ export type ValidationResult = { }; export type ValidationError = { - type: 'missing_column' | 'type_mismatch' | 'constraint_mismatch' | 'missing_table'; + type: + | 'missing_column' + | 'type_mismatch' + | 'constraint_mismatch' + | 'missing_table'; message: string; location: { component: string; @@ -788,16 +839,19 @@ export type ValidationWarning = { export const validateMigrationsAgainstSchema = ( component: SchemaComponent, - options?: { strict: boolean } + options?: { strict: boolean }, ): ValidationResult => { const result: ValidationResult = { valid: true, errors: [], - warnings: [] + warnings: [], }; // Only validate if component has both TypeScript definitions AND migrations - if (!hasTypeScriptDefinitions(component) || component.migrations.length === 0) { + if ( + !hasTypeScriptDefinitions(component) || + component.migrations.length === 0 + ) { return result; } @@ -815,10 +869,10 @@ export const validateMigrationsAgainstSchema = ( location: { component: component.schemaComponentKey, file: mismatch.file, - line: mismatch.line + line: mismatch.line, }, expected: mismatch.expected, - actual: mismatch.actual + actual: mismatch.actual, }); result.valid = false; } @@ -871,7 +925,7 @@ export class MigrationOrchestrator { constructor( private database: DatabaseSchemaComponent, private snapshotPath: string = '.dumbo/snapshots', - private migrationsPath: string = '.dumbo/migrations' + private migrationsPath: string = '.dumbo/migrations', ) {} async processMigrations(): Promise { @@ -885,29 +939,40 @@ export class MigrationOrchestrator { } // Process user-provided migrations - const userProvidedMigrations = await this.processUserProvidedMigrations(snapshot); + const userProvidedMigrations = + await this.processUserProvidedMigrations(snapshot); // Generate migrations for TypeScript-defined components - const generatedMigrations = await this.generateMigrationsFromDefinitions(snapshot); + const generatedMigrations = + await this.generateMigrationsFromDefinitions(snapshot); return { userProvided: userProvidedMigrations, generated: generatedMigrations, - all: [...userProvidedMigrations, ...generatedMigrations] + all: [...userProvidedMigrations, ...generatedMigrations], }; } - private async processUserProvidedMigrations(snapshot: DatabaseSnapshot | null): Promise { + private async processUserProvidedMigrations( + snapshot: DatabaseSnapshot | null, + ): Promise { const newMigrations: SQLMigration[] = []; // Process database-level migrations if (this.database.migrations.length > 0) { const snapshotMigrations = snapshot?.migrations || []; - const newDbMigrations = getNewMigrations(this.database.migrations, snapshotMigrations); + const newDbMigrations = getNewMigrations( + this.database.migrations, + snapshotMigrations, + ); for (const migration of newDbMigrations) { // Write to migrations folder - await this.writeMigrationToFile(migration, 'database', this.database.schemaComponentKey); + await this.writeMigrationToFile( + migration, + 'database', + this.database.schemaComponentKey, + ); newMigrations.push(migration); } } @@ -919,10 +984,17 @@ export class MigrationOrchestrator { // Process schema migrations if (schema.migrations.length > 0) { const snapshotMigrations = schemaSnapshot?.migrations || []; - const newSchemaMigrations = getNewMigrations(schema.migrations, snapshotMigrations); + const newSchemaMigrations = getNewMigrations( + schema.migrations, + snapshotMigrations, + ); for (const migration of newSchemaMigrations) { - await this.writeMigrationToFile(migration, 'schema', schema.schemaComponentKey); + await this.writeMigrationToFile( + migration, + 'schema', + schema.schemaComponentKey, + ); newMigrations.push(migration); } } @@ -933,10 +1005,17 @@ export class MigrationOrchestrator { if (table.migrations.length > 0) { const snapshotMigrations = tableSnapshot?.migrations || []; - const newTableMigrations = getNewMigrations(table.migrations, snapshotMigrations); + const newTableMigrations = getNewMigrations( + table.migrations, + snapshotMigrations, + ); for (const migration of newTableMigrations) { - await this.writeMigrationToFile(migration, 'table', table.schemaComponentKey); + await this.writeMigrationToFile( + migration, + 'table', + table.schemaComponentKey, + ); newMigrations.push(migration); } } @@ -949,7 +1028,7 @@ export class MigrationOrchestrator { private async writeMigrationToFile( migration: SQLMigration, type: 'database' | 'schema' | 'table', - componentKey: string + componentKey: string, ): Promise { const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `${timestamp}-${migration.name}.sql`; @@ -961,21 +1040,24 @@ export class MigrationOrchestrator { `-- Migration name: ${migration.name}`, `-- Written at: ${new Date().toISOString()}`, '', - '' + '', ].join('\n'); - const content = header + migration.sqls.map(sql => sql.toString()).join(';\n') + ';'; + const content = + header + migration.sqls.map((sql) => sql.toString()).join(';\n') + ';'; await writeFile(`${this.migrationsPath}/${filename}`, content); } - private async validateMigrations(snapshot: DatabaseSnapshot | null): Promise { + private async validateMigrations( + snapshot: DatabaseSnapshot | null, + ): Promise { if (!snapshot) return { valid: true }; // Validate database migrations const dbValidation = validateMigrationImmutability( this.database.migrations, - snapshot.migrations || [] + snapshot.migrations || [], ); if (!dbValidation.valid) return dbValidation; @@ -986,7 +1068,7 @@ export class MigrationOrchestrator { const schemaValidation = validateMigrationImmutability( schema.migrations, - schemaSnapshot.migrations || [] + schemaSnapshot.migrations || [], ); if (!schemaValidation.valid) return schemaValidation; @@ -996,7 +1078,7 @@ export class MigrationOrchestrator { const tableValidation = validateMigrationImmutability( table.migrations, - tableSnapshot.migrations || [] + tableSnapshot.migrations || [], ); if (!tableValidation.valid) return tableValidation; } @@ -1025,7 +1107,9 @@ export class MigrationOrchestrator { private async loadSnapshot(): Promise { try { // Load root snapshot which imports all nested snapshots - const module = await import(`${this.snapshotPath}/${this.database.databaseName}.snapshot.ts`); + const module = await import( + `${this.snapshotPath}/${this.database.databaseName}.snapshot.ts` + ); return module.databaseSnapshot; } catch (error) { // No snapshot found, will generate everything from scratch @@ -1038,7 +1122,9 @@ export class MigrationOrchestrator { await this.generateDatabaseSnapshot(this.database); } - private async generateDatabaseSnapshot(database: DatabaseSchemaComponent): Promise { + private async generateDatabaseSnapshot( + database: DatabaseSchemaComponent, + ): Promise { // Generate nested snapshot files for (const schema of database.schemas.values()) { await this.generateSchemaSnapshot(schema); @@ -1046,7 +1132,10 @@ export class MigrationOrchestrator { // Generate root database snapshot that imports schemas const imports = Array.from(database.schemas.values()) - .map(s => `import { ${s.schemaName}Snapshot } from './schemas/${s.schemaName}.snapshot';`) + .map( + (s) => + `import { ${s.schemaName}Snapshot } from './schemas/${s.schemaName}.snapshot';`, + ) .join('\n'); const migrationsStr = this.formatMigrationsForSnapshot(database.migrations); @@ -1059,16 +1148,21 @@ export const databaseSnapshot = { timestamp: '${new Date().toISOString()}', schemas: { ${Array.from(database.schemas.values()) - .map(s => `${s.schemaName}: ${s.schemaName}Snapshot`) + .map((s) => `${s.schemaName}: ${s.schemaName}Snapshot`) .join(',\n ')} }, migrations: ${migrationsStr} } as const;`; - await writeFile(`${this.snapshotPath}/${database.databaseName}.snapshot.ts`, content); + await writeFile( + `${this.snapshotPath}/${database.databaseName}.snapshot.ts`, + content, + ); } - private async generateSchemaSnapshot(schema: DatabaseSchemaSchemaComponent): Promise { + private async generateSchemaSnapshot( + schema: DatabaseSchemaSchemaComponent, + ): Promise { // Generate table snapshots first for (const table of schema.tables.values()) { await this.generateTableSnapshot(schema.schemaName, table); @@ -1076,7 +1170,10 @@ export const databaseSnapshot = { // Generate schema snapshot that imports tables const imports = Array.from(schema.tables.values()) - .map(t => `import { ${t.tableName}Snapshot } from './${schema.schemaName}/${t.tableName}.snapshot';`) + .map( + (t) => + `import { ${t.tableName}Snapshot } from './${schema.schemaName}/${t.tableName}.snapshot';`, + ) .join('\n'); const migrationsStr = this.formatMigrationsForSnapshot(schema.migrations); @@ -1087,16 +1184,22 @@ export const ${schema.schemaName}Snapshot = { schemaName: '${schema.schemaName}', tables: { ${Array.from(schema.tables.values()) - .map(t => `${t.tableName}: ${t.tableName}Snapshot`) + .map((t) => `${t.tableName}: ${t.tableName}Snapshot`) .join(',\n ')} }, migrations: ${migrationsStr} } as const;`; - await writeFile(`${this.snapshotPath}/schemas/${schema.schemaName}.snapshot.ts`, content); + await writeFile( + `${this.snapshotPath}/schemas/${schema.schemaName}.snapshot.ts`, + content, + ); } - private async generateTableSnapshot(schemaName: string, table: TableSchemaComponent): Promise { + private async generateTableSnapshot( + schemaName: string, + table: TableSchemaComponent, + ): Promise { const migrationsStr = this.formatMigrationsForSnapshot(table.migrations); const content = `export const ${table.tableName}Snapshot = { @@ -1114,14 +1217,21 @@ export const ${schema.schemaName}Snapshot = { migrations: ${migrationsStr} } as const;`; - await writeFile(`${this.snapshotPath}/schemas/${schemaName}/${table.tableName}.snapshot.ts`, content); + await writeFile( + `${this.snapshotPath}/schemas/${schemaName}/${table.tableName}.snapshot.ts`, + content, + ); } - private formatMigrationsForSnapshot(migrations: ReadonlyArray): string { + private formatMigrationsForSnapshot( + migrations: ReadonlyArray, + ): string { if (migrations.length === 0) return '[]'; - const migrationStrs = migrations.map(m => { - const sqlsStr = m.sqls.map(sql => `'${sql.toString().replace(/'/g, "\\'")}'`).join(', '); + const migrationStrs = migrations.map((m) => { + const sqlsStr = m.sqls + .map((sql) => `'${sql.toString().replace(/'/g, "\\'")}'`) + .join(', '); return `{ name: '${m.name}', sqls: [${sqlsStr}] }`; }); @@ -1130,10 +1240,12 @@ export const ${schema.schemaName}Snapshot = { private groupMigrations(migrations: SQLMigration[]) { return { - schemas: migrations.filter(m => m.name.includes('schema')), - tables: migrations.filter(m => m.name.includes('create_table')), - columns: migrations.filter(m => m.name.includes('add_') || m.name.includes('alter_')), - indexes: migrations.filter(m => m.name.includes('index')), + schemas: migrations.filter((m) => m.name.includes('schema')), + tables: migrations.filter((m) => m.name.includes('create_table')), + columns: migrations.filter( + (m) => m.name.includes('add_') || m.name.includes('alter_'), + ), + indexes: migrations.filter((m) => m.name.includes('index')), }; } } @@ -1157,8 +1269,12 @@ describe('generateTableMigrations', () => { tableName: 'users', columns: { id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255, nullable: false }) - } + email: column('email', { + type: 'varchar', + length: 255, + nullable: false, + }), + }, }); const migrations = table.generateMigrations(); @@ -1173,23 +1289,26 @@ describe('generateTableMigrations', () => { const snapshot = { tableName: 'users', columns: { - id: { type: 'serial' as const, primaryKey: true } + id: { type: 'serial' as const, primaryKey: true }, }, - indexes: {} + indexes: {}, }; const table = tableSchemaComponent({ tableName: 'users', columns: { id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }) - } + email: column('email', { type: 'varchar', length: 255 }), + }, }); const migrations = table.generateMigrations(snapshot); assert.equal(migrations.length, 1); - assert.match(migrations[0].sqls[0], /ALTER TABLE users ADD COLUMN IF NOT EXISTS email/); + assert.match( + migrations[0].sqls[0], + /ALTER TABLE users ADD COLUMN IF NOT EXISTS email/, + ); }); }); ``` @@ -1220,19 +1339,22 @@ describe('MigrationOrchestrator integration', () => { tableName: 'users', columns: { id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }) - } + email: column('email', { type: 'varchar', length: 255 }), + }, }), posts: tableSchemaComponent({ tableName: 'posts', columns: { id: column('id', { type: 'serial', primaryKey: true }), - userId: column('userId', { type: 'int', references: { table: 'users', column: 'id' } }) - } - }) - } - }) - } + userId: column('userId', { + type: 'int', + references: { table: 'users', column: 'id' }, + }), + }, + }), + }, + }), + }, }); const orchestrator = new MigrationOrchestrator(schema); @@ -1245,7 +1367,10 @@ describe('MigrationOrchestrator integration', () => { WHERE table_schema = 'public' `); - assert.deepEqual(tables.map(t => t.table_name).sort(), ['posts', 'users']); + assert.deepEqual(tables.map((t) => t.table_name).sort(), [ + 'posts', + 'users', + ]); // Verify columns exist const columns = await db.query(` @@ -1273,7 +1398,7 @@ describe('Schema evolution E2E', () => { // 2. Schema modification const v2Schema = createSchema({ version: 2, - addColumn: { table: 'users', column: 'phone' } + addColumn: { table: 'users', column: 'phone' }, }); // 3. Diff detection @@ -1314,7 +1439,7 @@ const usersTable = tableSchemaComponent({ tableName: 'users', columns: { id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }) + email: column('email', { type: 'varchar', length: 255 }), }, migrations: [ sqlMigration('001_custom_create', [ @@ -1322,9 +1447,9 @@ const usersTable = tableSchemaComponent({ id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}' - )` - ]) - ] + )`, + ]), + ], }); ``` @@ -1363,15 +1488,17 @@ export const usersSnapshot = { tableName: 'users', columns: { id: { type: 'serial' as const, primaryKey: true }, - email: { type: 'varchar' as const, length: 255, nullable: false } + email: { type: 'varchar' as const, length: 255, nullable: false }, }, indexes: {}, migrations: [ { name: '001_custom_create', - sqls: ['CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT \'{}\')'] - } - ] + sqls: [ + "CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}')", + ], + }, + ], } as const; ``` @@ -1384,7 +1511,7 @@ const usersTable = tableSchemaComponent({ columns: { id: column('id', { type: 'serial', primaryKey: true }), email: column('email', { type: 'varchar', length: 255 }), - name: column('name', { type: 'varchar', length: 100 }) // NEW! + name: column('name', { type: 'varchar', length: 100 }), // NEW! }, migrations: [ sqlMigration('001_custom_create', [ @@ -1392,12 +1519,13 @@ const usersTable = tableSchemaComponent({ id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}' - )` + )`, ]), - sqlMigration('002_add_name', [ // NEW! - SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)` - ]) - ] + sqlMigration('002_add_name', [ + // NEW! + SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)`, + ]), + ], }); ``` @@ -1423,12 +1551,12 @@ const usersTable = tableSchemaComponent({ id BIGSERIAL PRIMARY KEY, // Changed from SERIAL! email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}' - )` + )`, ]), sqlMigration('002_add_name', [ - SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)` - ]) - ] + SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)`, + ]), + ], }); await orchestrator.processMigrations(); @@ -1452,8 +1580,11 @@ const postsTable = tableSchemaComponent({ columns: { id: column('id', { type: 'serial', primaryKey: true }), title: column('title', { type: 'varchar', length: 255 }), - userId: column('userId', { type: 'int', references: { table: 'users', column: 'id' } }) - } + userId: column('userId', { + type: 'int', + references: { table: 'users', column: 'id' }, + }), + }, // No migrations provided - will be generated! }); @@ -1565,6 +1696,7 @@ Each phase builds on the previous one and can be tested independently. The key i This plan unifies SQL migrations and TypeScript schema definitions through: ### Core Innovations + 1. **Dual-purpose snapshots** - Track both current schema state AND component migrations 2. **Migration diffing** - Detect new user-provided migrations by comparing with snapshots 3. **Immutability enforcement** - Prevent dangerous modification of existing migrations @@ -1572,6 +1704,7 @@ This plan unifies SQL migrations and TypeScript schema definitions through: 5. **Nested snapshot structure** - Minimize git conflicts by separating tables into files ### Key Benefits + - **Flexibility** - Support SQL-first, TypeScript-first, or hybrid approaches - **Safety** - Migration immutability prevents accidental schema corruption - **Type Safety** - Generate types from snapshots for Pongo query builders @@ -1579,6 +1712,7 @@ This plan unifies SQL migrations and TypeScript schema definitions through: - **Clear Audit Trail** - Track all migrations with source metadata ### Migration Workflow + 1. Component defines schema (TypeScript) and/or migrations (SQL) 2. Orchestrator validates immutability against snapshot 3. New user-provided migrations are written to files @@ -1594,4 +1728,4 @@ This architecture provides a robust foundation for schema evolution while mainta - Migration bubbling happens at lines 79-83 in schemaComponent.ts - Recent refactoring in commit 03fb40a changed to use `Exclude` instead of `Omit` - Test patterns follow existing dumboSchema.unit.spec.ts structure -- Integration target is Pongo schema system in src/packages/pongo/src/core/schema/index.ts \ No newline at end of file +- Integration target is Pongo schema system in src/packages/pongo/src/core/schema/index.ts diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index c96e6592..b5d8ee36 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -1,3 +1,4 @@ +import type { SQLColumnToken } from '../../sql'; import { schemaComponent, type SchemaComponent, @@ -22,7 +23,8 @@ export const columnSchemaComponent = ({ ...migrationsOrComponents }: { columnName: string; -} & SchemaComponentOptions): ColumnSchemaComponent => { +} & SchemaComponentOptions & + Omit): ColumnSchemaComponent => { const sc = schemaComponent( ColumnURN({ name: columnName }), migrationsOrComponents, diff --git a/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts index 8508c8a9..6dc80afd 100644 --- a/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts @@ -1,13 +1,9 @@ import { - mapSchemaComponentsOfType, schemaComponent, type SchemaComponent, type SchemaComponentOptions, } from '../schemaComponent'; -import { - columnSchemaComponent, - type ColumnSchemaComponent, -} from './columnSchemaComponent'; +import { type ColumnSchemaComponent } from './columnSchemaComponent'; export type IndexURNType = 'sc:dumbo:index'; export type IndexURN = `${IndexURNType}:${string}`; @@ -16,11 +12,9 @@ export type IndexSchemaComponent = SchemaComponent< IndexURN, Readonly<{ indexName: string; - columns: ReadonlyMap; + columnNames: ReadonlyArray; isUnique: boolean; - addColumn: ( - column: string | ColumnSchemaComponent, - ) => ColumnSchemaComponent; + addColumn: (column: string | ColumnSchemaComponent) => void; }> >; @@ -38,31 +32,19 @@ export const indexSchemaComponent = ({ columnNames: string[]; isUnique: boolean; } & SchemaComponentOptions): IndexSchemaComponent => { - const columns = columnNames.map((columnName) => - columnSchemaComponent({ columnName }), - ); - const sc = schemaComponent(IndexURN({ name: indexName }), { migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...columns], + components: [...(migrationsOrComponents.components ?? [])], }); return { ...sc, indexName, - get columns() { - return mapSchemaComponentsOfType( - sc.components, - IndexURNType, - (c) => c.columnName, - ); + get columnNames() { + return columnNames; }, addColumn: (column: string | ColumnSchemaComponent) => - sc.addComponent( - typeof column === 'string' - ? columnSchemaComponent({ columnName: column }) - : column, - ), + columnNames.push(typeof column === 'string' ? column : column.columnName), isUnique, }; }; diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 300e31c3..425066c4 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -6,7 +6,6 @@ import { } from '../schemaComponent'; import { ColumnURNType, - columnSchemaComponent, type ColumnSchemaComponent, } from './columnSchemaComponent'; import { @@ -27,24 +26,20 @@ export type TableSchemaComponent = SchemaComponent< tableName: string; columns: ReadonlyMap; indexes: ReadonlyMap; - addColumn: ( - column: string | ColumnSchemaComponent, - ) => ColumnSchemaComponent; + addColumn: (column: ColumnSchemaComponent) => ColumnSchemaComponent; addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; }> >; export const tableSchemaComponent = ({ tableName, - columnNames, + columns, ...migrationsOrComponents }: { tableName: string; - columnNames?: string[]; + columns?: ColumnSchemaComponent[]; } & SchemaComponentOptions): TableSchemaComponent => { - const columns = - columnNames?.map((columnName) => columnSchemaComponent({ columnName })) ?? - []; + columns ??= []; const base = schemaComponent(TableURN({ name: tableName }), { migrations: migrationsOrComponents.migrations ?? [], @@ -68,12 +63,7 @@ export const tableSchemaComponent = ({ (c) => c.indexName, ); }, - addColumn: (column: string | ColumnSchemaComponent) => - base.addComponent( - typeof column === 'string' - ? columnSchemaComponent({ columnName: column }) - : column, - ), + addColumn: (column: ColumnSchemaComponent) => base.addComponent(column), addIndex: (index: IndexSchemaComponent) => base.addComponent(index), }; }; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 0b5885cb..b4a1845d 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,3 +1,4 @@ +import type { SQLColumnToken } from '../../sql'; import { columnSchemaComponent, type ColumnSchemaComponent, @@ -21,7 +22,7 @@ const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = ( name: string, - options?: SchemaComponentOptions, + options: SchemaComponentOptions & Omit, ): ColumnSchemaComponent => columnSchemaComponent({ columnName: name, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 77451815..97ce6c0e 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -1,33 +1,43 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; +import { SQL } from '../../sql'; import { dumboSchema } from './index'; +const { database, schema, table, column, index } = dumboSchema; +const { Varchar } = SQL.column.type; + void describe('dumboSchema', () => { void it('should create a column', () => { - const col = dumboSchema.column('id'); + const col = column('id', { + type: Varchar('max'), + }); assert.strictEqual(col.columnName, 'id'); }); void it('should create an index', () => { - const idx = dumboSchema.index('idx_email', ['email']); + const idx = index('idx_email', ['email']); assert.strictEqual(idx.indexName, 'idx_email'); assert.strictEqual(idx.isUnique, false); }); void it('should create a unique index', () => { - const idx = dumboSchema.index('idx_email', ['email'], { unique: true }); + const idx = index('idx_email', ['email'], { unique: true }); assert.strictEqual(idx.indexName, 'idx_email'); assert.strictEqual(idx.isUnique, true); }); void it('should create a table with columns and indexes', () => { - const tbl = dumboSchema.table('users', { + const tbl = table('users', { columns: { - id: dumboSchema.column('id'), - email: dumboSchema.column('email'), + id: column('id', { + type: Varchar('max'), + }), + email: column('email', { + type: Varchar('max'), + }), }, indexes: { - idx_email: dumboSchema.index('idx_email', ['email']), + idx_email: index('idx_email', ['email']), }, }); @@ -40,10 +50,12 @@ void describe('dumboSchema', () => { }); void it('should create a named schema', () => { - const sch = dumboSchema.schema('public', { - users: dumboSchema.table('users', { + const sch = schema('public', { + users: table('users', { columns: { - id: dumboSchema.column('id'), + id: column('id', { + type: Varchar('max'), + }), }, }), }); @@ -54,10 +66,12 @@ void describe('dumboSchema', () => { }); void it('should create a default schema without name', () => { - const sch = dumboSchema.schema({ - users: dumboSchema.table('users', { + const sch = schema({ + users: table('users', { columns: { - id: dumboSchema.column('id'), + id: column('id', { + type: Varchar('max'), + }), }, }), }); @@ -67,11 +81,13 @@ void describe('dumboSchema', () => { }); void it('should create a default database', () => { - const db = dumboSchema.database({ - public: dumboSchema.schema('public', { - users: dumboSchema.table('users', { + const db = database({ + public: schema('public', { + users: table('users', { columns: { - id: dumboSchema.column('id'), + id: column('id', { + type: Varchar('max'), + }), }, }), }), @@ -83,11 +99,13 @@ void describe('dumboSchema', () => { }); void it('should create a named database', () => { - const db = dumboSchema.database('myapp', { - public: dumboSchema.schema('public', { - users: dumboSchema.table('users', { + const db = database('myapp', { + public: schema('public', { + users: table('users', { columns: { - id: dumboSchema.column('id'), + id: column('id', { + type: Varchar('max'), + }), }, }), }), @@ -99,12 +117,14 @@ void describe('dumboSchema', () => { }); void it('should handle DEFAULT_SCHEMA', () => { - const db = dumboSchema.database( + const db = database( 'myapp', - dumboSchema.schema({ - users: dumboSchema.table('users', { + schema({ + users: table('users', { columns: { - id: dumboSchema.column('id'), + id: column('id', { + type: Varchar('max'), + }), }, }), }), @@ -116,13 +136,13 @@ void describe('dumboSchema', () => { }); void it('should create schema from table names', () => { - const sch = dumboSchema.schema.from('public', ['users', 'posts']); + const sch = schema.from('public', ['users', 'posts']); assert.strictEqual(sch.schemaName, 'public'); assert.strictEqual(sch.tables.size, 2); }); void it('should create database from schema names', () => { - const db = dumboSchema.database.from('myapp', ['public', 'analytics']); + const db = database.from('myapp', ['public', 'analytics']); assert.strictEqual(db.databaseName, 'myapp'); assert.strictEqual(db.schemas.size, 2); }); @@ -131,40 +151,54 @@ void describe('dumboSchema', () => { // Samples // Simple database with tables in default schema -export const simpleDb = dumboSchema.database( - 'myapp', - dumboSchema.schema({ - users: dumboSchema.table('users', { - columns: { - id: dumboSchema.column('id'), - email: dumboSchema.column('email'), - name: dumboSchema.column('name'), - }, - indexes: { - idx_email: dumboSchema.index('idx_email', ['email'], { - unique: true, - }), - }, + +const users = table('users', { + columns: { + id: column('id', { + type: Varchar('max'), }), + email: column('email', { + type: Varchar('max'), + }), + name: column('name', { + type: Varchar('max'), + }), + }, +}); + +export const simpleDb = database( + 'myapp', + schema({ + users, }), ); // Database with multiple schemas -const multiSchemaDb = dumboSchema.database('myapp', { - public: dumboSchema.schema('public', { - users: dumboSchema.table('users', { +const multiSchemaDb = database('myapp', { + public: schema('public', { + users: table('users', { columns: { - id: dumboSchema.column('id'), - email: dumboSchema.column('email'), + id: column('id', { + type: Varchar('max'), + }), + email: column('email', { + type: Varchar('max'), + }), }, }), }), - analytics: dumboSchema.schema('analytics', { - events: dumboSchema.table('events', { + analytics: schema('analytics', { + events: table('events', { columns: { - id: dumboSchema.column('id'), - user_id: dumboSchema.column('user_id'), - timestamp: dumboSchema.column('timestamp'), + id: column('id', { + type: Varchar('max'), + }), + user_id: column('user_id', { + type: Varchar('max'), + }), + timestamp: column('timestamp', { + type: Varchar('max'), + }), }, }), }), diff --git a/src/packages/dumbo/src/core/schema/schema_feature.md b/src/packages/dumbo/src/core/schema/schema_feature.md index afa5b776..9de5d3a3 100644 --- a/src/packages/dumbo/src/core/schema/schema_feature.md +++ b/src/packages/dumbo/src/core/schema/schema_feature.md @@ -33,15 +33,18 @@ A feature is an opaque schema component that encapsulates internal components (t // Base marker interface for all features export type FeatureSchemaComponent< URN extends string = string, - AdditionalData extends Record = Record -> = SchemaComponent; // Hidden from user API -}>; + AdditionalData extends Record = Record, +> = SchemaComponent< + URN, + AdditionalData & { + __featureMarker: true; + internalComponents: ReadonlyMap; // Hidden from user API + } +>; // Type guard export const isFeatureSchemaComponent = ( - comp: AnySchemaComponent + comp: AnySchemaComponent, ): comp is FeatureSchemaComponent => { return '__featureMarker' in comp && comp.__featureMarker === true; }; @@ -60,37 +63,38 @@ export type EventStoreSchemaComponent = FeatureSchemaComponent< } >; -export const eventStoreSchema = ( - options?: { - inlineProjections?: Record; - } -): EventStoreSchemaComponent => { +export const eventStoreSchema = (options?: { + inlineProjections?: Record; +}): EventStoreSchemaComponent => { // Create internal tables (opaque to user) const streams = dumboTable('streams', { - migrations: [sqlMigration('create_streams', [streamsTableSQL])] + migrations: [sqlMigration('create_streams', [streamsTableSQL])], }); const messages = dumboTable('messages', { - migrations: [sqlMigration('create_messages', [messagesTableSQL])] + migrations: [sqlMigration('create_messages', [messagesTableSQL])], }); const subscriptions = dumboTable('subscriptions', { - migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])] + migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], }); // Create functions const appendFunction = dumboFunction('emt_append_to_stream', { - migrations: [sqlMigration('create_append_function', [appendToStreamSQL])] + migrations: [sqlMigration('create_append_function', [appendToStreamSQL])], }); // Handle inline projections (create tables from Pongo collections) const projectionTables = Object.entries(options?.inlineProjections ?? {}).map( - ([name, collectionSchema]) => dumboTable(collectionSchema.name, { - // Auto-generate table structure for Pongo collection - migrations: [sqlMigration(`create_projection_${name}`, [ - // Generate CREATE TABLE with _id and document columns - ])] - }) + ([name, collectionSchema]) => + dumboTable(collectionSchema.name, { + // Auto-generate table structure for Pongo collection + migrations: [ + sqlMigration(`create_projection_${name}`, [ + // Generate CREATE TABLE with _id and document columns + ]), + ], + }), ); const allComponents = [ @@ -98,7 +102,7 @@ export const eventStoreSchema = ( messages, subscriptions, appendFunction, - ...projectionTables + ...projectionTables, ]; const base = schemaComponent('sc:dumbo:feature:event_store', { @@ -110,7 +114,9 @@ export const eventStoreSchema = ( __featureMarker: true as const, eventStoreName: 'event_store', inlineProjections: options?.inlineProjections, - internalComponents: new Map(allComponents.map(c => [c.schemaComponentKey, c])), + internalComponents: new Map( + allComponents.map((c) => [c.schemaComponentKey, c]), + ), }; }; ``` @@ -128,21 +134,23 @@ export type PongoCollectionsSchemaComponent = FeatureSchemaComponent< >; export const pongoCollectionsSchema = ( - collections: Record + collections: Record, ): PongoCollectionsSchemaComponent => { // Create Dumbo table for each Pongo collection const tables = Object.entries(collections).map(([_name, collectionSchema]) => dumboTable(collectionSchema.name, { - migrations: [sqlMigration(`create_collection_${collectionSchema.name}`, [ - // Auto-generate table structure: - // CREATE TABLE {name} ( - // _id TEXT PRIMARY KEY, - // document JSONB NOT NULL, - // created_at TIMESTAMP DEFAULT NOW(), - // updated_at TIMESTAMP DEFAULT NOW() - // ) - ])] - }) + migrations: [ + sqlMigration(`create_collection_${collectionSchema.name}`, [ + // Auto-generate table structure: + // CREATE TABLE {name} ( + // _id TEXT PRIMARY KEY, + // document JSONB NOT NULL, + // created_at TIMESTAMP DEFAULT NOW(), + // updated_at TIMESTAMP DEFAULT NOW() + // ) + ]), + ], + }), ); const base = schemaComponent('sc:dumbo:feature:pongo_collections', { @@ -153,7 +161,7 @@ export const pongoCollectionsSchema = ( ...base, __featureMarker: true as const, pongoSchema: pongoSchema.db(collections), - internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), + internalComponents: new Map(tables.map((t) => [t.schemaComponentKey, t])), }; }; ``` @@ -182,56 +190,71 @@ export function systemSchema(options: { schemaIntrospection?: boolean; }): SystemSchemaComponent; export function systemSchema( - nameOrOptions?: string | { - migrationTracking?: boolean; - pongoMetadata?: boolean; - schemaIntrospection?: boolean; - } + nameOrOptions?: + | string + | { + migrationTracking?: boolean; + pongoMetadata?: boolean; + schemaIntrospection?: boolean; + }, ): SystemSchemaComponent | DatabaseSchemaSchemaComponent { - const options = typeof nameOrOptions === 'string' - ? { schemaName: nameOrOptions } - : nameOrOptions; + const options = + typeof nameOrOptions === 'string' + ? { schemaName: nameOrOptions } + : nameOrOptions; const tables: TableSchemaComponent[] = []; if (options?.migrationTracking !== false) { - tables.push(dumboTable('__migrations', { - migrations: [sqlMigration('create_migrations_table', [ - SQL`CREATE TABLE IF NOT EXISTS __migrations ( + tables.push( + dumboTable('__migrations', { + migrations: [ + sqlMigration('create_migrations_table', [ + SQL`CREATE TABLE IF NOT EXISTS __migrations ( id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, applied_at TIMESTAMP DEFAULT NOW() - )` - ])] - })); + )`, + ]), + ], + }), + ); } if (options?.pongoMetadata !== false) { - tables.push(dumboTable('__pongo_collections', { - migrations: [sqlMigration('create_pongo_metadata_table', [ - SQL`CREATE TABLE IF NOT EXISTS __pongo_collections ( + tables.push( + dumboTable('__pongo_collections', { + migrations: [ + sqlMigration('create_pongo_metadata_table', [ + SQL`CREATE TABLE IF NOT EXISTS __pongo_collections ( collection_name TEXT PRIMARY KEY, json_schema JSONB, created_at TIMESTAMP DEFAULT NOW(), updated_at TIMESTAMP DEFAULT NOW() - )` - ])] - })); + )`, + ]), + ], + }), + ); } if (options?.schemaIntrospection !== false) { - tables.push(dumboTable('__schema_metadata', { - migrations: [sqlMigration('create_schema_metadata_table', [ - SQL`CREATE TABLE IF NOT EXISTS __schema_metadata ( + tables.push( + dumboTable('__schema_metadata', { + migrations: [ + sqlMigration('create_schema_metadata_table', [ + SQL`CREATE TABLE IF NOT EXISTS __schema_metadata ( component_type TEXT NOT NULL, component_name TEXT NOT NULL, component_key TEXT PRIMARY KEY, definition JSONB NOT NULL, created_at TIMESTAMP DEFAULT NOW(), updated_at TIMESTAMP DEFAULT NOW() - )` - ])] - })); + )`, + ]), + ], + }), + ); } const base = schemaComponent('sc:dumbo:feature:system', { @@ -246,7 +269,7 @@ export function systemSchema( pongoMetadata: options?.pongoMetadata !== false, schemaIntrospection: options?.schemaIntrospection !== false, }, - internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), + internalComponents: new Map(tables.map((t) => [t.schemaComponentKey, t])), } satisfies SystemSchemaComponent; // If schemaName provided, wrap in schema component @@ -258,7 +281,7 @@ export function systemSchema( } export const isSystemSchemaComponent = ( - comp: AnySchemaComponent + comp: AnySchemaComponent, ): comp is SystemSchemaComponent => { return comp.schemaComponentKey === 'sc:dumbo:feature:system'; }; @@ -284,7 +307,7 @@ export const dumboFunction = ( name: string, options: SchemaComponentOptions & { language?: 'plpgsql' | 'sql' | 'javascript'; - } + }, ): FunctionSchemaComponent => { const base = schemaComponent(`sc:dumbo:function:${name}` as FunctionURN, { migrations: options.migrations ?? [], @@ -311,15 +334,15 @@ const simpleApp = dumboDatabase('my_app', { columns: { id: dumboColumn('id', { type: 'serial', primaryKey: true }), email: dumboColumn('email', { type: 'varchar', length: 255 }), - } + }, }), posts: dumboTable('posts', { columns: { id: dumboColumn('id', { type: 'serial', primaryKey: true }), userId: dumboColumn('user_id', { type: 'int' }), - } + }, }), - }) + }), }); // System tables (__migrations, __pongo_collections, __schema_metadata) @@ -340,16 +363,16 @@ const hotelApp = dumboDatabase('hotel', { columns: { id: dumboColumn('id', { type: 'serial', primaryKey: true }), name: dumboColumn('name', { type: 'varchar', length: 255 }), - } + }, }), reservations: dumboTable('reservations', { columns: { id: dumboColumn('id', { type: 'serial', primaryKey: true }), guestId: dumboColumn('guest_id', { type: 'int' }), - } + }, }), - }) + }), }); // Generated migrations include: @@ -374,8 +397,12 @@ const hotelApp = dumboDatabase('hotel', { // Read models in separate schema read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), + guests: dumboTable('guests', { + /* ... */ + }), + reservations: dumboTable('reservations', { + /* ... */ + }), }), }); @@ -398,7 +425,7 @@ const hotelApp = dumboDatabase('hotel', { inlineProjections: { guestSummary: pongoSchema.collection('guest_summary'), reservationSummary: pongoSchema.collection('reservation_summary'), - } + }, }), }), @@ -423,59 +450,81 @@ const hotelApp = dumboDatabase('hotel', { ```typescript // Database group for organizing related databases -const hotelSystemGroup = databaseGroup('hotel_system', { - // Operational database - operational: dumboDatabase('hotel_operational', { - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), +const hotelSystemGroup = databaseGroup( + 'hotel_system', + { + // Operational database + operational: dumboDatabase('hotel_operational', { + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { + /* ... */ + }), + reservations: dumboTable('reservations', { + /* ... */ + }), + }), }), - }), - // Analytics database - analytics: dumboDatabase('hotel_analytics', { - public: dumboDatabaseSchema('public', { - events: dumboTable('events', { /* ... */ }), - metrics: dumboTable('metrics', { /* ... */ }), + // Analytics database + analytics: dumboDatabase('hotel_analytics', { + public: dumboDatabaseSchema('public', { + events: dumboTable('events', { + /* ... */ + }), + metrics: dumboTable('metrics', { + /* ... */ + }), + }), }), - }), - // Reporting database - reporting: dumboDatabase('hotel_reporting', { - public: dumboDatabaseSchema('public', { - reports: dumboTable('reports', { /* ... */ }), + // Reporting database + reporting: dumboDatabase('hotel_reporting', { + public: dumboDatabaseSchema('public', { + reports: dumboTable('reports', { + /* ... */ + }), + }), }), - }), -}, { - // Shared system schema across all databases - shared: { - systemSchema: systemSchema('shared_system'), - } -}); + }, + { + // Shared system schema across all databases + shared: { + systemSchema: systemSchema('shared_system'), + }, + }, +); ``` ## System Schema Placement Strategy ### Single Schema + When database has exactly one schema, system tables are automatically added to that schema. ```typescript const db = dumboDatabase('app', { - public: dumboDatabaseSchema('public', { /* ... */ }) + public: dumboDatabaseSchema('public', { + /* ... */ + }), }); // ✅ System tables → 'public' schema ``` ### Multiple Schemas Without Explicit System Schema + Default behavior: Use database type's default schema convention. ```typescript const db = dumboDatabase('app', { - event_store: dumboDatabaseSchema('event_store', { /* ... */ }), - read_models: dumboDatabaseSchema('read_models', { /* ... */ }), + event_store: dumboDatabaseSchema('event_store', { + /* ... */ + }), + read_models: dumboDatabaseSchema('read_models', { + /* ... */ + }), }); // PostgreSQL: Check for 'public' schema, otherwise use first schema @@ -487,14 +536,19 @@ const db = dumboDatabase('app', { ``` ### Explicit System Schema + User can explicitly place system schema anywhere: ```typescript // Option A: Dedicated system schema const db = dumboDatabase('app', { system: systemSchema('admin'), // Returns DatabaseSchemaSchemaComponent - event_store: dumboDatabaseSchema('event_store', { /* ... */ }), - read_models: dumboDatabaseSchema('read_models', { /* ... */ }), + event_store: dumboDatabaseSchema('event_store', { + /* ... */ + }), + read_models: dumboDatabaseSchema('read_models', { + /* ... */ + }), }); // Option B: System schema within existing schema @@ -503,7 +557,9 @@ const db = dumboDatabase('app', { system: systemSchema(), // Just the component eventStore: eventStoreSchema(), }), - read_models: dumboDatabaseSchema('read_models', { /* ... */ }), + read_models: dumboDatabaseSchema('read_models', { + /* ... */ + }), }); ``` @@ -577,23 +633,31 @@ export const shardingStrategies = { #### Schema-Level Sharding (Multi-Tenancy) ```typescript -const hotelApp = dumboDatabase('hotel', { - system: systemSchema('system'), // Not sharded +const hotelApp = dumboDatabase( + 'hotel', + { + system: systemSchema('system'), // Not sharded - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), + event_store: dumboDatabaseSchema('event_store', { + eventStore: eventStoreSchema(), + }), - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), - }), -}, { - sharding: { - level: 'schema', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott', 'hyatt']), - } -}); + read_models: dumboDatabaseSchema('read_models', { + guests: dumboTable('guests', { + /* ... */ + }), + reservations: dumboTable('reservations', { + /* ... */ + }), + }), + }, + { + sharding: { + level: 'schema', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott', 'hyatt']), + }, + }, +); // PostgreSQL generates: // CREATE SCHEMA system; -- Not sharded @@ -611,15 +675,23 @@ const hotelApp = dumboDatabase('hotel', { #### Database-Level Sharding ```typescript -const hotelApp = dumboDatabase('hotel', { - event_store: dumboDatabaseSchema('event_store', { /* ... */ }), - read_models: dumboDatabaseSchema('read_models', { /* ... */ }), -}, { - sharding: { - level: 'database', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), - } -}); +const hotelApp = dumboDatabase( + 'hotel', + { + event_store: dumboDatabaseSchema('event_store', { + /* ... */ + }), + read_models: dumboDatabaseSchema('read_models', { + /* ... */ + }), + }, + { + sharding: { + level: 'database', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + }, + }, +); // PostgreSQL generates separate databases: // CREATE DATABASE hilton_hotel; @@ -633,18 +705,30 @@ const hotelApp = dumboDatabase('hotel', { #### Table-Level Sharding (Time Partitioning) ```typescript -const analyticsDb = dumboDatabase('analytics', { - public: dumboDatabaseSchema('public', { - events: dumboTable('events', { /* ... */ }), - metrics: dumboTable('metrics', { /* ... */ }), - }), -}, { - sharding: { - level: 'table', - strategy: shardingStrategies.timePartition(['2024_01', '2024_02', '2024_03']), - applyTo: ['events'], // Only shard events table - } -}); +const analyticsDb = dumboDatabase( + 'analytics', + { + public: dumboDatabaseSchema('public', { + events: dumboTable('events', { + /* ... */ + }), + metrics: dumboTable('metrics', { + /* ... */ + }), + }), + }, + { + sharding: { + level: 'table', + strategy: shardingStrategies.timePartition([ + '2024_01', + '2024_02', + '2024_03', + ]), + applyTo: ['events'], // Only shard events table + }, + }, +); // Generates: // CREATE TABLE events_2024_01 (...); @@ -656,19 +740,29 @@ const analyticsDb = dumboDatabase('analytics', { #### Database Group Sharding ```typescript -const hotelSystemGroup = databaseGroup('hotel_system', { - operational: dumboDatabase('hotel_operational', { /* ... */ }), - analytics: dumboDatabase('hotel_analytics', { /* ... */ }), - reporting: dumboDatabase('hotel_reporting', { /* ... */ }), -}, { - sharding: { - level: 'database', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), +const hotelSystemGroup = databaseGroup( + 'hotel_system', + { + operational: dumboDatabase('hotel_operational', { + /* ... */ + }), + analytics: dumboDatabase('hotel_analytics', { + /* ... */ + }), + reporting: dumboDatabase('hotel_reporting', { + /* ... */ + }), }, - shared: { - systemSchema: systemSchema('shared_system'), - } -}); + { + sharding: { + level: 'database', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + }, + shared: { + systemSchema: systemSchema('shared_system'), + }, + }, +); // Generates for each tenant: // Tenant 'hilton': @@ -691,25 +785,35 @@ const hotelSystemGroup = databaseGroup('hotel_system', { const hotelApp = dumboDatabase('hotel', { system: systemSchema('system'), // No sharding - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }, { - // Override: shard event store by tenant - sharding: { - level: 'schema', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), - } - }), - - analytics: dumboDatabaseSchema('analytics', { - events: dumboTable('events', { /* ... */ }), - }, { - // Override: shard analytics by region - sharding: { - level: 'table', - strategy: shardingStrategies.region(['us_east', 'us_west', 'eu']), - } - }), + event_store: dumboDatabaseSchema( + 'event_store', + { + eventStore: eventStoreSchema(), + }, + { + // Override: shard event store by tenant + sharding: { + level: 'schema', + strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), + }, + }, + ), + + analytics: dumboDatabaseSchema( + 'analytics', + { + events: dumboTable('events', { + /* ... */ + }), + }, + { + // Override: shard analytics by region + sharding: { + level: 'table', + strategy: shardingStrategies.region(['us_east', 'us_west', 'eu']), + }, + }, + ), }); // Generates: @@ -728,7 +832,10 @@ const hotelApp = dumboDatabase('hotel', { ```typescript export type DatabaseGroup< - Databases extends Record = Record + Databases extends Record = Record< + string, + DatabaseSchemaComponent + >, > = { groupName: string; databases: Databases; @@ -742,7 +849,9 @@ export type DatabaseGroup< }; }; -export const databaseGroup = >( +export const databaseGroup = < + T extends Record, +>( groupName: string, databases: T, options?: { @@ -750,7 +859,7 @@ export const databaseGroup = > shared?: { systemSchema?: SystemSchemaComponent; }; - } + }, ): DatabaseGroup => ({ groupName, databases, @@ -780,13 +889,15 @@ const hotelSchema = dumboDatabase('hotel', { eventStore: eventStoreSchema(), }), read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), + guests: dumboTable('guests', { + /* ... */ + }), }), }); // PostgreSQL migration generation const pgMigrations = generateMigrations(hotelSchema, { - databaseType: 'postgresql' + databaseType: 'postgresql', }); // Generates: // CREATE SCHEMA event_store; @@ -797,7 +908,7 @@ const pgMigrations = generateMigrations(hotelSchema, { // SQLite migration generation const sqliteMigrations = generateMigrations(hotelSchema, { databaseType: 'sqlite', - sqliteStrategy: 'prefix-tables' // or 'separate-files' + sqliteStrategy: 'prefix-tables', // or 'separate-files' }); // prefix-tables generates: // CREATE TABLE event_store_streams (...); @@ -854,8 +965,10 @@ While features are opaque at definition time, generated types expose internal st const hotelSchema = dumboDatabase('hotel', { public: dumboDatabaseSchema('public', { eventStore: eventStoreSchema(), - guests: dumboTable('guests', { /* ... */ }), - }) + guests: dumboTable('guests', { + /* ... */ + }), + }), }); // Generated types expose all tables (including event store internals) @@ -953,8 +1066,12 @@ const hotelSchema = dumboDatabase('hotel', { }), read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), + guests: dumboTable('guests', { + /* ... */ + }), + reservations: dumboTable('reservations', { + /* ... */ + }), }), }); @@ -965,39 +1082,44 @@ const deployment = { mapping: { // Map Emmett components to Dumbo schemas/tables 'event-store': 'event_store.eventStore', - 'guests': 'read_models.guests', - 'reservations': 'read_models.reservations', - } + guests: 'read_models.guests', + reservations: 'read_models.reservations', + }, }; ``` ## Design Decisions ### 1. Why Feature Components Extend SchemaComponent? + - **Uniform Composition**: Features compose like tables, indexes, etc. - **Consistent API**: Same `addComponent()` pattern everywhere - **Type Safety**: Generic `SchemaComponent` infrastructure works for features - **Migration System**: Features participate in migration collection automatically ### 2. Why System Schema is a Feature Component? + - **Consistent Placement**: Same composition rules as other features - **Flexible Location**: Can be in dedicated schema or mixed with application schemas -- **Opaque Internals**: Users don't need to know about __migrations, __pongo_collections tables +- **Opaque Internals**: Users don't need to know about **migrations, **pongo_collections tables - **Customizable**: Power users can configure which system tables to include ### 3. Why Generic Sharding vs. Hardcoded Multi-Tenancy? + - **Extensibility**: Supports time partitioning, regional sharding, custom strategies - **Composition**: Can combine multiple sharding dimensions in the future - **Flexibility**: Same mechanism for different use cases (multi-tenancy, scaling, compliance) - **Simplicity**: Single concept to learn instead of multiple special cases ### 4. Why Database Groups? + - **Organization**: Large systems have multiple related databases - **Shared Resources**: System schema can be shared across databases - **Consistent Sharding**: Apply same tenant strategy to command/query/event databases - **Documentation**: Architecture diagrams show logical database groupings ### 5. Why Portable Schema Definitions? + - **Developer Experience**: Write once, deploy to PostgreSQL or SQLite - **Testing**: Test with SQLite, deploy to PostgreSQL - **Flexibility**: Change database type without rewriting schema @@ -1015,7 +1137,7 @@ Support multiple sharding dimensions: { level: 'database', strategy: shardingStrategies.multiTenant(['hilton']) }, { level: 'schema', strategy: shardingStrategies.region(['us', 'eu']) }, { level: 'table', strategy: shardingStrategies.timePartition(['2024_01']) }, - ] + ]; } // Generates: Database hilton_hotel → Schema us_event_store → Table events_2024_01 ``` @@ -1057,7 +1179,7 @@ Support foreign keys across sharded databases: const relationship = crossDatabaseForeignKey( 'read_models.reservations.guest_id', 'read_models.guests.id', - { onDelete: 'CASCADE' } + { onDelete: 'CASCADE' }, ); ``` @@ -1066,16 +1188,26 @@ const relationship = crossDatabaseForeignKey( Explicit migration dependencies for complex scenarios: ```typescript -const createUsersMigration = sqlMigration('create_users', [/* ... */]); -const createPostsMigration = sqlMigration('create_posts', [/* ... */], { - dependsOn: [createUsersMigration], // Posts table needs users table first -}); +const createUsersMigration = sqlMigration('create_users', [ + /* ... */ +]); +const createPostsMigration = sqlMigration( + 'create_posts', + [ + /* ... */ + ], + { + dependsOn: [createUsersMigration], // Posts table needs users table first + }, +); ``` ## Open Questions ### Q1: Function Component Placement + Should functions be: + - Top-level components in schema (like tables)? - Nested within table components? - Only within feature components? @@ -1083,7 +1215,9 @@ Should functions be: **Current decision**: Top-level and within features (flexible) ### Q2: SQLite Multiple Schema Strategy Default + For SQLite with multiple schemas, default to: + - Table prefixing (simpler, single file) - Separate database files (stronger isolation) - Error (force explicit choice) @@ -1091,7 +1225,9 @@ For SQLite with multiple schemas, default to: **Current decision**: Table prefixing (simpler default, user can override) ### Q3: System Schema Naming Convention + Should system schema default name be: + - `__dumbo_system` (clear it's framework) - `_system` (shorter) - `system` (clean but might conflict) @@ -1099,20 +1235,25 @@ Should system schema default name be: **Current decision**: `system` for dedicated schema, auto-add to first schema otherwise ### Q4: Sharding Naming Pattern Syntax + Support only string templates, or also functions? + - String: `'{shard}_{resource}'` - Function: `(shard, resource) => ...` **Current decision**: Both (string for simplicity, function for flexibility) ### Q5: Feature Component Registration + Should there be a registry for custom feature components? ```typescript // Register custom feature registerFeatureComponent('custom_feature', { detect: (comp) => comp.schemaComponentKey.startsWith('sc:custom:'), - extractMigrations: (comp) => { /* ... */ }, + extractMigrations: (comp) => { + /* ... */ + }, }); ``` @@ -1121,6 +1262,7 @@ registerFeatureComponent('custom_feature', { ## Implementation Checklist ### Phase 1: Core Feature Components + - [ ] `FeatureSchemaComponent` base type - [ ] `isFeatureSchemaComponent()` type guard - [ ] `EventStoreSchemaComponent` implementation @@ -1129,12 +1271,14 @@ registerFeatureComponent('custom_feature', { - [ ] `FunctionSchemaComponent` implementation ### Phase 2: System Schema Placement + - [ ] Auto-detection logic for single schema - [ ] Database type default schema logic (PostgreSQL 'public', etc.) - [ ] Explicit system schema placement - [ ] System schema component tests ### Phase 3: Generic Sharding + - [ ] `ShardingStrategy` type definition - [ ] `ShardingConfig` type definition - [ ] Predefined strategies (`multiTenant`, `region`, `timePartition`) @@ -1144,6 +1288,7 @@ registerFeatureComponent('custom_feature', { - [ ] Exclusion logic for shared resources ### Phase 4: Database Groups + - [ ] `DatabaseGroup` type definition - [ ] `databaseGroup()` factory function - [ ] Shared system schema across databases @@ -1151,6 +1296,7 @@ registerFeatureComponent('custom_feature', { - [ ] Migration generation for groups ### Phase 5: Migration Generation + - [ ] Detect feature components in migration collector - [ ] Extract internal migrations from features - [ ] Database-specific migration generation (PostgreSQL vs SQLite) @@ -1159,6 +1305,7 @@ registerFeatureComponent('custom_feature', { - [ ] Sharding-aware migration generation ### Phase 6: Type Generation + - [ ] Extract internal structure from features - [ ] Generate types for event store tables - [ ] Generate types for Pongo collections @@ -1166,6 +1313,7 @@ registerFeatureComponent('custom_feature', { - [ ] Exclude system tables from main types (optional) ### Phase 7: Integration & Documentation + - [ ] Dumbo schema API updates - [ ] Pongo schema integration - [ ] Emmett architecture mapping examples From c1fb729f217cae42a81614c70f36e91919140409 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 13 Nov 2025 11:16:35 +0100 Subject: [PATCH 21/39] Adjusted column schema definition to include mandatory type --- .../core/schema/dumboSchema/dumboSchema.ts | 7 +- .../dumboSchema/dumboSchema.unit.spec.ts | 64 +++++-------------- 2 files changed, 21 insertions(+), 50 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index b4a1845d..642a2153 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,4 +1,4 @@ -import type { SQLColumnToken } from '../../sql'; +import type { SQLColumnToken, SQLToken } from '../../sql'; import { columnSchemaComponent, type ColumnSchemaComponent, @@ -22,10 +22,13 @@ const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = ( name: string, - options: SchemaComponentOptions & Omit, + type: string | SQLToken, + options: SchemaComponentOptions & + Omit = {}, ): ColumnSchemaComponent => columnSchemaComponent({ columnName: name, + type, ...options, }); diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 97ce6c0e..04a918fc 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -8,9 +8,7 @@ const { Varchar } = SQL.column.type; void describe('dumboSchema', () => { void it('should create a column', () => { - const col = column('id', { - type: Varchar('max'), - }); + const col = column('id', Varchar('max')); assert.strictEqual(col.columnName, 'id'); }); @@ -29,12 +27,8 @@ void describe('dumboSchema', () => { void it('should create a table with columns and indexes', () => { const tbl = table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), - email: column('email', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), }, indexes: { idx_email: index('idx_email', ['email']), @@ -53,9 +47,7 @@ void describe('dumboSchema', () => { const sch = schema('public', { users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), }, }), }); @@ -69,9 +61,7 @@ void describe('dumboSchema', () => { const sch = schema({ users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), }, }), }); @@ -85,9 +75,7 @@ void describe('dumboSchema', () => { public: schema('public', { users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), }, }), }), @@ -103,9 +91,7 @@ void describe('dumboSchema', () => { public: schema('public', { users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), }, }), }), @@ -122,9 +108,7 @@ void describe('dumboSchema', () => { schema({ users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), }, }), }), @@ -154,15 +138,9 @@ void describe('dumboSchema', () => { const users = table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), - email: column('email', { - type: Varchar('max'), - }), - name: column('name', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + name: column('name', Varchar('max')), }, }); @@ -178,27 +156,17 @@ const multiSchemaDb = database('myapp', { public: schema('public', { users: table('users', { columns: { - id: column('id', { - type: Varchar('max'), - }), - email: column('email', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), }, }), }), analytics: schema('analytics', { events: table('events', { columns: { - id: column('id', { - type: Varchar('max'), - }), - user_id: column('user_id', { - type: Varchar('max'), - }), - timestamp: column('timestamp', { - type: Varchar('max'), - }), + id: column('id', Varchar('max')), + userId: column('user_id', Varchar('max')), + timestamp: column('timestamp', Varchar('max')), }, }), }), From e48c1dd80399830cc3d0a21e9beb733869933270 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 13 Nov 2025 11:51:11 +0100 Subject: [PATCH 22/39] Renamed SQLColumnTokens to SQLColumnTypeTokens to make it explict --- .../dumbo/src/core/sql/processors/columnProcessors.ts | 6 +++--- src/packages/dumbo/src/core/sql/sql.ts | 8 ++++---- src/packages/dumbo/src/core/sql/tokens/columnTokens.ts | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts b/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts index 78766385..9245fa8b 100644 --- a/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts +++ b/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts @@ -1,9 +1,9 @@ import type { BigIntegerToken, DefaultSQLColumnToken, + SQLColumnTypeTokens, SQLToken, } from '../tokens'; -import { SQLColumnTokens } from '../tokens'; import { SQLProcessor, type SQLProcessorContext } from './sqlProcessor'; type ExtractTokenType = T extends (...args: never[]) => infer R @@ -15,8 +15,8 @@ type ExtractTokenType = T extends (...args: never[]) => infer R : never; export type DefaultSQLColumnProcessors = { - [key in keyof SQLColumnTokens]: SQLProcessor< - ExtractTokenType<(typeof SQLColumnTokens)[key]> + [key in keyof SQLColumnTypeTokens]: SQLProcessor< + ExtractTokenType<(typeof SQLColumnTypeTokens)[key]> >; }; diff --git a/src/packages/dumbo/src/core/sql/sql.ts b/src/packages/dumbo/src/core/sql/sql.ts index e4f317e7..2588acfb 100644 --- a/src/packages/dumbo/src/core/sql/sql.ts +++ b/src/packages/dumbo/src/core/sql/sql.ts @@ -8,7 +8,7 @@ import type { ParametrizedSQL } from './parametrizedSQL'; import { isTokenizedSQL, TokenizedSQL } from './tokenizedSQL'; import { SQLColumnToken, - SQLColumnTokens, + SQLColumnTypeTokens, SQLIdentifier, SQLIn, SQLPlain, @@ -99,10 +99,10 @@ SQL.check = { isSQLIn: SQLIn.check, }; -const columnFactory: SQLColumnToken & { type: typeof SQLColumnTokens } = +const columnFactory: SQLColumnToken & { type: typeof SQLColumnTypeTokens } = SQLColumnToken as unknown as SQLColumnToken & { - type: typeof SQLColumnTokens; + type: typeof SQLColumnTypeTokens; }; -columnFactory.type = SQLColumnTokens; +columnFactory.type = SQLColumnTypeTokens; SQL.column = columnFactory; diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index 34b06247..d619d0a3 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -67,7 +67,7 @@ export type AutoIncrementSQLColumnToken = SQLToken< export const AutoIncrementSQLColumnToken = SQLToken('SQL_COLUMN_AUTO_INCREMENT'); -export const SQLColumnTokens = { +export const SQLColumnTypeTokens = { AutoIncrement: AutoIncrementSQLColumnToken.from, BigInteger: BigIntegerToken.from(undefined!), BigSerial: BigSerialToken.from(undefined!), @@ -78,7 +78,7 @@ export const SQLColumnTokens = { Timestamptz: TimestamptzToken.from(undefined!), Varchar: VarcharToken.from, }; -export type SQLColumnTokens = typeof SQLColumnTokens; +export type SQLColumnTypeTokens = typeof SQLColumnTypeTokens; export type DefaultSQLColumnToken = | AutoIncrementSQLColumnToken From bc24ce62d7790196dd5f8e7df4e48f052f8c0900 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Fri, 14 Nov 2025 15:16:08 +0100 Subject: [PATCH 23/39] Refactored SQLToken to be flat to make easier integration with Schema --- .../components/columnSchemaComponent.ts | 2 +- .../core/schema/dumboSchema/dumboSchema.ts | 6 +- .../core/sql/processors/columnProcessors.ts | 24 +--- .../core/sql/processors/defaultProcessors.ts | 2 +- .../src/core/sql/processors/sqlProcessor.ts | 8 +- .../sql/processors/sqlProcessorRegistry.ts | 6 +- src/packages/dumbo/src/core/sql/sql.ts | 14 ++- .../src/core/sql/tokenizedSQL/tokenizedSQL.ts | 16 ++- .../dumbo/src/core/sql/tokens/columnTokens.ts | 104 ++++++++++-------- .../dumbo/src/core/sql/tokens/sqlToken.ts | 84 ++++++++++---- .../core/sql/processors/columProcessors.ts | 6 +- .../core/sql/processors/columProcessors.ts | 6 +- 12 files changed, 162 insertions(+), 116 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index b5d8ee36..c98254dd 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -24,7 +24,7 @@ export const columnSchemaComponent = ({ }: { columnName: string; } & SchemaComponentOptions & - Omit): ColumnSchemaComponent => { + Omit): ColumnSchemaComponent => { const sc = schemaComponent( ColumnURN({ name: columnName }), migrationsOrComponents, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 642a2153..7590b046 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,4 +1,4 @@ -import type { SQLColumnToken, SQLToken } from '../../sql'; +import type { AnySQLToken, SQLColumnToken } from '../../sql'; import { columnSchemaComponent, type ColumnSchemaComponent, @@ -22,9 +22,9 @@ const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = ( name: string, - type: string | SQLToken, + type: string | AnySQLToken, options: SchemaComponentOptions & - Omit = {}, + Omit = {}, ): ColumnSchemaComponent => columnSchemaComponent({ columnName: name, diff --git a/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts b/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts index 9245fa8b..20f6af40 100644 --- a/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts +++ b/src/packages/dumbo/src/core/sql/processors/columnProcessors.ts @@ -1,23 +1,9 @@ -import type { - BigIntegerToken, - DefaultSQLColumnToken, - SQLColumnTypeTokens, - SQLToken, -} from '../tokens'; +import type { BigIntegerToken, DefaultSQLColumnToken } from '../tokens'; +import { AutoIncrementSQLColumnToken, SQLColumnTypeTokens } from '../tokens'; import { SQLProcessor, type SQLProcessorContext } from './sqlProcessor'; -type ExtractTokenType = T extends (...args: never[]) => infer R - ? R extends SQLToken - ? R - : never - : T extends SQLToken - ? T - : never; - export type DefaultSQLColumnProcessors = { - [key in keyof SQLColumnTypeTokens]: SQLProcessor< - ExtractTokenType<(typeof SQLColumnTypeTokens)[key]> - >; + [key in keyof SQLColumnTypeTokens]: SQLProcessor; }; export const mapDefaultSQLColumnProcessors = ( @@ -26,9 +12,9 @@ export const mapDefaultSQLColumnProcessors = ( context: SQLProcessorContext, ) => void, ): DefaultSQLColumnProcessors => ({ - AutoIncrement: SQLProcessor({ + AutoIncrement: SQLProcessor({ canHandle: 'SQL_COLUMN_AUTO_INCREMENT', - handle: (token, context) => { + handle: (token: AutoIncrementSQLColumnToken, context) => { mapColumnType(token, context); }, }), diff --git a/src/packages/dumbo/src/core/sql/processors/defaultProcessors.ts b/src/packages/dumbo/src/core/sql/processors/defaultProcessors.ts index e95f5868..215dad9f 100644 --- a/src/packages/dumbo/src/core/sql/processors/defaultProcessors.ts +++ b/src/packages/dumbo/src/core/sql/processors/defaultProcessors.ts @@ -17,7 +17,7 @@ export const ExpandSQLInProcessor: SQLProcessor = SQLProcessor({ canHandle: 'SQL_IN', handle: (token: SQLIn, context: SQLProcessorContext) => { const { builder, mapper, processorsRegistry } = context; - const { values: inValues, column } = token.value; + const { values: inValues, column } = token; if (inValues.value.length === 0) { builder.addParam(mapper.mapValue(false)); diff --git a/src/packages/dumbo/src/core/sql/processors/sqlProcessor.ts b/src/packages/dumbo/src/core/sql/processors/sqlProcessor.ts index d3ea4936..268282d3 100644 --- a/src/packages/dumbo/src/core/sql/processors/sqlProcessor.ts +++ b/src/packages/dumbo/src/core/sql/processors/sqlProcessor.ts @@ -1,5 +1,5 @@ import type { ParametrizedSQLBuilder } from '../parametrizedSQL'; -import type { SQLToken } from '../tokens'; +import type { AnySQLToken } from '../tokens'; import type { SQLValueMapper } from '../valueMappers'; import type { SQLProcessorsReadonlyRegistry } from './sqlProcessorRegistry'; @@ -9,7 +9,7 @@ export type SQLProcessorContext = { processorsRegistry: SQLProcessorsReadonlyRegistry; }; -export type SQLProcessor = { +export type SQLProcessor = { canHandle: Token['sqlTokenType']; handle: (value: Token, context: SQLProcessorContext) => void; }; @@ -17,11 +17,11 @@ export type SQLProcessor = { // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnySQLProcessor = SQLProcessor; -export type SQLProcessorOptions = { +export type SQLProcessorOptions = { canHandle: Token['sqlTokenType']; handle: (value: Token, context: SQLProcessorContext) => void; }; -export const SQLProcessor = ( +export const SQLProcessor = ( options: SQLProcessorOptions, ): SQLProcessor => options; diff --git a/src/packages/dumbo/src/core/sql/processors/sqlProcessorRegistry.ts b/src/packages/dumbo/src/core/sql/processors/sqlProcessorRegistry.ts index fb1faf66..7a560e69 100644 --- a/src/packages/dumbo/src/core/sql/processors/sqlProcessorRegistry.ts +++ b/src/packages/dumbo/src/core/sql/processors/sqlProcessorRegistry.ts @@ -1,8 +1,8 @@ -import type { SQLToken } from '../tokens'; +import type { AnySQLToken } from '../tokens'; import type { AnySQLProcessor, SQLProcessor } from './sqlProcessor'; export interface SQLProcessorsReadonlyRegistry { - get( + get( tokenType: Token['sqlTokenType'], ): SQLProcessor | null; all(): ReadonlyMap; @@ -46,7 +46,7 @@ export const SQLProcessorsRegistry = (options?: { const registry = { register, - get: ( + get: ( tokenType: string, ): SQLProcessor | null => { return processors.get(tokenType) ?? null; diff --git a/src/packages/dumbo/src/core/sql/sql.ts b/src/packages/dumbo/src/core/sql/sql.ts index 2588acfb..170bff24 100644 --- a/src/packages/dumbo/src/core/sql/sql.ts +++ b/src/packages/dumbo/src/core/sql/sql.ts @@ -8,7 +8,7 @@ import type { ParametrizedSQL } from './parametrizedSQL'; import { isTokenizedSQL, TokenizedSQL } from './tokenizedSQL'; import { SQLColumnToken, - SQLColumnTypeTokens, + SQLColumnTypeTokensFactory, SQLIdentifier, SQLIn, SQLPlain, @@ -99,10 +99,12 @@ SQL.check = { isSQLIn: SQLIn.check, }; -const columnFactory: SQLColumnToken & { type: typeof SQLColumnTypeTokens } = - SQLColumnToken as unknown as SQLColumnToken & { - type: typeof SQLColumnTypeTokens; - }; -columnFactory.type = SQLColumnTypeTokens; +const columnFactory: typeof SQLColumnToken.from & { + type: typeof SQLColumnTypeTokensFactory; +} = SQLColumnToken.from as unknown as typeof SQLColumnToken.from & { + type: typeof SQLColumnTypeTokensFactory; +}; +columnFactory.type = + SQLColumnTypeTokensFactory as unknown as typeof SQLColumnTypeTokensFactory; SQL.column = columnFactory; diff --git a/src/packages/dumbo/src/core/sql/tokenizedSQL/tokenizedSQL.ts b/src/packages/dumbo/src/core/sql/tokenizedSQL/tokenizedSQL.ts index c8dcd802..ac663c59 100644 --- a/src/packages/dumbo/src/core/sql/tokenizedSQL/tokenizedSQL.ts +++ b/src/packages/dumbo/src/core/sql/tokenizedSQL/tokenizedSQL.ts @@ -1,14 +1,20 @@ -import { SQLArray, SQLLiteral, SQLPlain, SQLToken } from '../tokens'; +import { + SQLArray, + SQLLiteral, + SQLPlain, + SQLToken, + type AnySQLToken, +} from '../tokens'; export type TokenizedSQL = Readonly<{ __brand: 'tokenized-sql'; sqlChunks: ReadonlyArray; - sqlTokens: ReadonlyArray; + sqlTokens: ReadonlyArray; }>; const TokenizedSQLBuilder = () => { const sqlChunks: string[] = []; - const sqlTokens: SQLToken[] = []; + const sqlTokens: AnySQLToken[] = []; return { addSQL(str: string): void { @@ -17,10 +23,10 @@ const TokenizedSQLBuilder = () => { addSQLs(str: ReadonlyArray): void { sqlChunks.push(...str); }, - addToken(value: SQLToken): void { + addToken(value: AnySQLToken): void { sqlTokens.push(value); }, - addTokens(vals: ReadonlyArray): void { + addTokens(vals: ReadonlyArray): void { sqlTokens.push(...vals); }, build(): TokenizedSQL { diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index d619d0a3..c675974b 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -1,55 +1,44 @@ -import { SQLToken } from './sqlToken'; +import { SQLToken, type AnySQLToken } from './sqlToken'; -export type SerialToken = SQLToken<'SQL_COLUMN_SERIAL', never>; -export const SerialToken = SQLToken( - 'SQL_COLUMN_SERIAL', - () => undefined!, -); +export type SerialToken = SQLToken<'SQL_COLUMN_SERIAL'>; +export const SerialToken = SQLToken('SQL_COLUMN_SERIAL'); -export type BigSerialToken = SQLToken<'SQL_COLUMN_BIGSERIAL', never>; -export const BigSerialToken = SQLToken( - 'SQL_COLUMN_BIGSERIAL', - () => undefined!, -); +export type BigSerialToken = SQLToken<'SQL_COLUMN_BIGSERIAL'>; +export const BigSerialToken = SQLToken('SQL_COLUMN_BIGSERIAL'); -export type IntegerToken = SQLToken<'SQL_COLUMN_INTEGER', never>; -export const IntegerToken = SQLToken( - 'SQL_COLUMN_INTEGER', - () => undefined!, -); +export type IntegerToken = SQLToken<'SQL_COLUMN_INTEGER'>; +export const IntegerToken = SQLToken('SQL_COLUMN_INTEGER'); -export type BigIntegerToken = SQLToken<'SQL_COLUMN_BIGINT', never>; -export const BigIntegerToken = SQLToken( - 'SQL_COLUMN_BIGINT', - () => undefined!, -); +export type BigIntegerToken = SQLToken<'SQL_COLUMN_BIGINT'>; +export const BigIntegerToken = SQLToken('SQL_COLUMN_BIGINT'); -export type JSONBToken = SQLToken<'SQL_COLUMN_JSONB', never>; -export const JSONBToken = SQLToken( - 'SQL_COLUMN_JSONB', - () => undefined!, -); +export type JSONBToken = SQLToken<'SQL_COLUMN_JSONB'>; +export const JSONBToken = SQLToken('SQL_COLUMN_JSONB'); -export type TimestampToken = SQLToken<'SQL_COLUMN_TIMESTAMP', never>; -export const TimestampToken = SQLToken( - 'SQL_COLUMN_TIMESTAMP', - () => undefined!, -); +export type TimestampToken = SQLToken<'SQL_COLUMN_TIMESTAMP'>; +export const TimestampToken = SQLToken('SQL_COLUMN_TIMESTAMP'); -export type TimestamptzToken = SQLToken<'SQL_COLUMN_TIMESTAMPTZ', never>; +export type TimestamptzToken = SQLToken<'SQL_COLUMN_TIMESTAMPTZ'>; export const TimestamptzToken = SQLToken( 'SQL_COLUMN_TIMESTAMPTZ', - () => undefined!, ); -export type VarcharToken = SQLToken<'SQL_COLUMN_VARCHAR', number | 'max'>; -export const VarcharToken = SQLToken('SQL_COLUMN_VARCHAR'); +export type VarcharToken = SQLToken< + 'SQL_COLUMN_VARCHAR', + { length: number | 'max' } +>; +export const VarcharToken = SQLToken( + 'SQL_COLUMN_VARCHAR', + (length) => ({ + length, + }), +); export type SQLColumnToken = SQLToken< 'SQL_COLUMN', { name: string; - type: ColumnType | SQLToken; + type: ColumnType | AnySQLToken; notNull?: boolean; unique?: boolean; primaryKey?: boolean; @@ -68,25 +57,48 @@ export const AutoIncrementSQLColumnToken = SQLToken('SQL_COLUMN_AUTO_INCREMENT'); export const SQLColumnTypeTokens = { + AutoIncrement: AutoIncrementSQLColumnToken, + BigInteger: BigIntegerToken, + BigSerial: BigSerialToken, + Integer: IntegerToken, + JSONB: JSONBToken, + Serial: SerialToken, + Timestamp: TimestampToken, + Timestamptz: TimestamptzToken, + Varchar: VarcharToken, +}; + +export type SQLColumnTypeTokens = { + AutoIncrement: AutoIncrementSQLColumnToken; + BigInteger: BigIntegerToken; + BigSerial: BigSerialToken; + Integer: IntegerToken; + JSONB: JSONBToken; + Serial: SerialToken; + Timestamp: TimestampToken; + Timestamptz: TimestamptzToken; + Varchar: VarcharToken; +}; + +export const SQLColumnTypeTokensFactory = { AutoIncrement: AutoIncrementSQLColumnToken.from, - BigInteger: BigIntegerToken.from(undefined!), - BigSerial: BigSerialToken.from(undefined!), - Integer: IntegerToken.from(undefined!), - JSONB: JSONBToken.from(undefined!), - Serial: SerialToken.from(undefined!), - Timestamp: TimestampToken.from(undefined!), - Timestamptz: TimestamptzToken.from(undefined!), + BigInteger: BigIntegerToken.from(), + BigSerial: BigSerialToken.from(), + Integer: IntegerToken.from(), + JSONB: JSONBToken.from(), + Serial: SerialToken.from(), + Timestamp: TimestampToken.from(), + Timestamptz: TimestamptzToken.from(), Varchar: VarcharToken.from, }; -export type SQLColumnTypeTokens = typeof SQLColumnTypeTokens; export type DefaultSQLColumnToken = | AutoIncrementSQLColumnToken - | BigIntegerToken + | SerialToken | BigSerialToken | IntegerToken | JSONBToken - | SerialToken + | BigIntegerToken | TimestampToken | TimestamptzToken | VarcharToken; diff --git a/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts b/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts index 0248a642..f8678ef2 100644 --- a/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts +++ b/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts @@ -1,28 +1,52 @@ export type SQLToken< TSymbol extends string = string, - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint, @typescript-eslint/no-explicit-any - TProps extends any = any, + TProps extends Omit, 'sqlTokenType'> | undefined = + | Omit, 'sqlTokenType'> + | undefined, > = { sqlTokenType: TSymbol; - value: TProps; -}; +} & (TProps extends undefined ? void : Omit); + +export type ExtractSQLTokenType = T extends (...args: never[]) => infer R + ? R extends SQLToken + ? R + : never + : T extends SQLToken + ? T + : never; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnySQLToken = SQLToken; export const SQLToken = < - SQLTokenType extends SQLToken, - TValue = SQLTokenType['value'], + SQLTokenType extends AnySQLToken, + TInput = keyof Omit extends never + ? void + : Omit, >( sqlTokenType: SQLTokenType['sqlTokenType'], - map?: (value: TValue) => SQLTokenType['value'], + map?: (input: TInput) => Omit, ) => { - const factory = (props: TValue): SQLTokenType => { - props = - map === undefined - ? (props as unknown as SQLTokenType['value']) - : map(props); + const factory = (input: TInput): SQLTokenType => { + let props: Omit; + + if (map !== undefined) { + props = map(input); + } else if (input === undefined || input === null) { + props = {} as Omit; + } else if (typeof input === 'object' && !Array.isArray(input)) { + // If input is already an object (but not array), spread it + props = input as Omit; + } else { + throw new Error( + `Cannot create SQLToken of type ${sqlTokenType} with input: ${input}`, + ); + } + return { sqlTokenType: sqlTokenType, [sqlTokenType]: true, - value: props, + ...props, } as unknown as SQLTokenType; }; @@ -32,20 +56,36 @@ export const SQLToken = < return { from: factory, check: check, type: sqlTokenType }; }; -SQLToken.check = (token: unknown): token is SQLToken => +SQLToken.check = ( + token: unknown, +): token is SQLTokenType => token !== null && typeof token === 'object' && 'sqlTokenType' in token; -export type SQLIdentifier = SQLToken<'SQL_IDENTIFIER', string>; -export const SQLIdentifier = SQLToken('SQL_IDENTIFIER'); +export type SQLIdentifier = SQLToken<'SQL_IDENTIFIER', { value: string }>; +export const SQLIdentifier = SQLToken( + 'SQL_IDENTIFIER', + (value) => ({ + value, + }), +); -export type SQLPlain = SQLToken<'SQL_RAW', string>; -export const SQLPlain = SQLToken('SQL_RAW'); +export type SQLPlain = SQLToken<'SQL_RAW', { value: string }>; +export const SQLPlain = SQLToken('SQL_RAW', (value) => ({ + value, +})); -export type SQLLiteral = SQLToken<'SQL_LITERAL', unknown>; -export const SQLLiteral = SQLToken('SQL_LITERAL'); +export type SQLLiteral = SQLToken<'SQL_LITERAL', { value: unknown }>; +export const SQLLiteral = SQLToken( + 'SQL_LITERAL', + (value) => ({ + value, + }), +); -export type SQLArray = SQLToken<'SQL_ARRAY', unknown[]>; -export const SQLArray = SQLToken('SQL_ARRAY'); +export type SQLArray = SQLToken<'SQL_ARRAY', { value: unknown[] }>; +export const SQLArray = SQLToken('SQL_ARRAY', (value) => ({ + value, +})); export type SQLIn = SQLToken< 'SQL_IN', diff --git a/src/packages/dumbo/src/storage/postgresql/core/sql/processors/columProcessors.ts b/src/packages/dumbo/src/storage/postgresql/core/sql/processors/columProcessors.ts index 11466237..a43c69f7 100644 --- a/src/packages/dumbo/src/storage/postgresql/core/sql/processors/columProcessors.ts +++ b/src/packages/dumbo/src/storage/postgresql/core/sql/processors/columProcessors.ts @@ -10,10 +10,10 @@ const mapColumnType = ( { builder }: SQLProcessorContext, ): void => { let columnSQL: string; - const { sqlTokenType, value } = token; + const { sqlTokenType } = token; switch (sqlTokenType) { case 'SQL_COLUMN_AUTO_INCREMENT': - columnSQL = `${value.bigint ? 'BIGSERIAL' : 'SERIAL'} ${value.primaryKey ? 'PRIMARY KEY' : ''}`; + columnSQL = `${token.bigint ? 'BIGSERIAL' : 'SERIAL'} ${token.primaryKey ? 'PRIMARY KEY' : ''}`; break; case 'SQL_COLUMN_BIGINT': columnSQL = 'BIGINT'; @@ -37,7 +37,7 @@ const mapColumnType = ( columnSQL = 'TIMESTAMPTZ'; break; case 'SQL_COLUMN_VARCHAR': - columnSQL = `VARCHAR ${Number.isNaN(value) ? '' : `(${value})`}`; + columnSQL = `VARCHAR ${Number.isNaN(token.length) ? '' : `(${token.length})`}`; break; default: { const exhaustiveCheck: never = sqlTokenType; diff --git a/src/packages/dumbo/src/storage/sqlite/core/sql/processors/columProcessors.ts b/src/packages/dumbo/src/storage/sqlite/core/sql/processors/columProcessors.ts index fbe6ccec..5a201a05 100644 --- a/src/packages/dumbo/src/storage/sqlite/core/sql/processors/columProcessors.ts +++ b/src/packages/dumbo/src/storage/sqlite/core/sql/processors/columProcessors.ts @@ -10,10 +10,10 @@ const mapColumnType = ( { builder }: SQLProcessorContext, ): void => { let columnSQL: string; - const { sqlTokenType, value } = token; + const { sqlTokenType } = token; switch (sqlTokenType) { case 'SQL_COLUMN_AUTO_INCREMENT': - columnSQL = `INTEGER ${value.primaryKey ? 'PRIMARY KEY' : ''} AUTOINCREMENT`; + columnSQL = `INTEGER ${token.primaryKey ? 'PRIMARY KEY' : ''} AUTOINCREMENT`; break; case 'SQL_COLUMN_BIGINT': columnSQL = 'INTEGER'; @@ -37,7 +37,7 @@ const mapColumnType = ( columnSQL = 'DATETIME'; break; case 'SQL_COLUMN_VARCHAR': - columnSQL = `VARCHAR ${Number.isNaN(value) ? '' : `(${value})`}`; + columnSQL = `VARCHAR ${Number.isNaN(token.length) ? '' : `(${token.length})`}`; break; default: { const exhaustiveCheck: never = sqlTokenType; From 511a4c8013dd79e93b908dc77988f83ae4fdb6c8 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 15 Nov 2025 10:14:57 +0100 Subject: [PATCH 24/39] Made column schema component be also SQLColumnToken --- .../src/core/schema/components/columnSchemaComponent.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index c98254dd..8402b8f9 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -17,9 +17,12 @@ export type ColumnSchemaComponent = SchemaComponent< Readonly<{ columnName: string; }> ->; +> & + SQLColumnToken; + export const columnSchemaComponent = ({ columnName, + type, ...migrationsOrComponents }: { columnName: string; @@ -33,5 +36,8 @@ export const columnSchemaComponent = ({ return { ...sc, columnName, + sqlTokenType: 'SQL_COLUMN', + name: columnName, + type, }; }; From 1b2b28d265c5235b60a4a6217af0f90091998d41 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 15 Nov 2025 12:56:36 +0100 Subject: [PATCH 25/39] Merged SQL helpers with dumbo schema --- src/packages/dumbo/src/core/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/packages/dumbo/src/core/index.ts b/src/packages/dumbo/src/core/index.ts index cb6df199..5ea06e77 100644 --- a/src/packages/dumbo/src/core/index.ts +++ b/src/packages/dumbo/src/core/index.ts @@ -6,6 +6,8 @@ import type { ExtractDumboDatabaseDriverOptions, InferDriverDatabaseType, } from './drivers'; +import { dumboSchema } from './schema'; +import { SQL, SQLColumnTypeTokensFactory } from './sql'; export * from './connections'; export * from './drivers'; @@ -39,3 +41,16 @@ export type DumboConnectionOptions< } & Omit : never : never; + +declare module './sql' { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace SQL { + export const columnN: typeof dumboSchema.column & { + type: typeof SQLColumnTypeTokensFactory; + }; + } +} +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access +(SQL as any).columnN = Object.assign(dumboSchema.column, { + type: SQLColumnTypeTokensFactory, +}); From 59b9f6d500431d16ce0132ea4b8126a5d1bd250c Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 15 Nov 2025 17:18:59 +0100 Subject: [PATCH 26/39] Added strong typing for column type in column schema --- .../core/schema/dumboSchema/dumboSchema.ts | 4 +- .../dumbo/src/core/sql/tokens/columnTokens.ts | 37 ++++++++++++------- .../dumbo/src/core/sql/tokens/sqlToken.ts | 1 + 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 7590b046..2f25c06b 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,4 +1,4 @@ -import type { AnySQLToken, SQLColumnToken } from '../../sql'; +import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; import { columnSchemaComponent, type ColumnSchemaComponent, @@ -22,7 +22,7 @@ const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = ( name: string, - type: string | AnySQLToken, + type: string | AnyColumnTypeToken, options: SchemaComponentOptions & Omit = {}, ): ColumnSchemaComponent => diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index c675974b..4d63d961 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -1,36 +1,47 @@ -import { SQLToken, type AnySQLToken } from './sqlToken'; +import { SQLToken } from './sqlToken'; -export type SerialToken = SQLToken<'SQL_COLUMN_SERIAL'>; +// TODO: Use URNs for sqltoken +export type ColumnTypeToken< + ColumnTypeName extends string = string, + TProps extends Omit, 'sqlTokenType'> | undefined = + | Omit, 'sqlTokenType'> + | undefined, +> = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps>; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyColumnTypeToken = ColumnTypeToken; + +export type SerialToken = ColumnTypeToken<'SERIAL'>; export const SerialToken = SQLToken('SQL_COLUMN_SERIAL'); -export type BigSerialToken = SQLToken<'SQL_COLUMN_BIGSERIAL'>; +export type BigSerialToken = ColumnTypeToken<'BIGSERIAL'>; export const BigSerialToken = SQLToken('SQL_COLUMN_BIGSERIAL'); -export type IntegerToken = SQLToken<'SQL_COLUMN_INTEGER'>; +export type IntegerToken = ColumnTypeToken<'INTEGER'>; export const IntegerToken = SQLToken('SQL_COLUMN_INTEGER'); -export type BigIntegerToken = SQLToken<'SQL_COLUMN_BIGINT'>; +export type BigIntegerToken = ColumnTypeToken<'BIGINT'>; export const BigIntegerToken = SQLToken('SQL_COLUMN_BIGINT'); -export type JSONBToken = SQLToken<'SQL_COLUMN_JSONB'>; +export type JSONBToken = ColumnTypeToken<'JSONB'>; export const JSONBToken = SQLToken('SQL_COLUMN_JSONB'); -export type TimestampToken = SQLToken<'SQL_COLUMN_TIMESTAMP'>; +export type TimestampToken = ColumnTypeToken<'TIMESTAMP'>; export const TimestampToken = SQLToken('SQL_COLUMN_TIMESTAMP'); -export type TimestamptzToken = SQLToken<'SQL_COLUMN_TIMESTAMPTZ'>; +export type TimestamptzToken = ColumnTypeToken<'TIMESTAMPTZ'>; export const TimestamptzToken = SQLToken( 'SQL_COLUMN_TIMESTAMPTZ', ); -export type VarcharToken = SQLToken< - 'SQL_COLUMN_VARCHAR', +export type VarcharToken = ColumnTypeToken< + 'VARCHAR', { length: number | 'max' } >; export const VarcharToken = SQLToken( 'SQL_COLUMN_VARCHAR', - (length) => ({ - length, + (length?: number | 'max') => ({ + length: length ?? 'max', }), ); @@ -38,7 +49,7 @@ export type SQLColumnToken = SQLToken< 'SQL_COLUMN', { name: string; - type: ColumnType | AnySQLToken; + type: ColumnType | AnyColumnTypeToken; notNull?: boolean; unique?: boolean; primaryKey?: boolean; diff --git a/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts b/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts index f8678ef2..e511ff50 100644 --- a/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts +++ b/src/packages/dumbo/src/core/sql/tokens/sqlToken.ts @@ -39,6 +39,7 @@ export const SQLToken = < props = input as Omit; } else { throw new Error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions `Cannot create SQLToken of type ${sqlTokenType} with input: ${input}`, ); } From f800f1ee9fd35a35764bd31f95a5560126693d02 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Sat, 15 Nov 2025 18:48:33 +0100 Subject: [PATCH 27/39] Added first version of stongly typed dumbo schema --- .../components/databaseSchemaComponent.ts | 45 +++++--- .../databaseSchemaSchemaComponent.ts | 37 +++++-- .../schema/components/tableSchemaComponent.ts | 45 +++++--- .../core/schema/dumboSchema/dumboSchema.ts | 101 +++++++++--------- .../dumboSchema/dumboSchema.unit.spec.ts | 8 +- 5 files changed, 141 insertions(+), 95 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts index 0f6da357..d79527c6 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts @@ -7,6 +7,7 @@ import { import { DatabaseSchemaURNType, databaseSchemaSchemaComponent, + type AnyDatabaseSchemaSchemaComponent, type DatabaseSchemaSchemaComponent, } from './databaseSchemaSchemaComponent'; @@ -17,44 +18,56 @@ export const DatabaseURNType: DatabaseURNType = 'sc:dumbo:database'; export const DatabaseURN = ({ name }: { name: string }): DatabaseURN => `${DatabaseURNType}:${name}`; -export type DatabaseSchemaComponent = SchemaComponent< +export type DatabaseSchemas< + Schemas extends + AnyDatabaseSchemaSchemaComponent = AnyDatabaseSchemaSchemaComponent, +> = Record; + +export type DatabaseSchemaComponent< + Schemas extends DatabaseSchemas = DatabaseSchemas, +> = SchemaComponent< DatabaseURN, Readonly<{ databaseName: string; - schemas: ReadonlyMap; + schemas: ReadonlyMap & Schemas; addSchema: ( schema: string | DatabaseSchemaSchemaComponent, ) => DatabaseSchemaSchemaComponent; }> >; -export const databaseSchemaComponent = ({ +export const databaseSchemaComponent = < + Schemas extends DatabaseSchemas = DatabaseSchemas, +>({ databaseName, - schemaNames, + schemas, ...migrationsOrComponents }: { databaseName: string; - schemaNames?: string[]; -} & SchemaComponentOptions): DatabaseSchemaComponent => { - const schemas = - schemaNames?.map((schemaName) => - databaseSchemaSchemaComponent({ schemaName }), - ) ?? []; + schemas?: Schemas; +} & SchemaComponentOptions): DatabaseSchemaComponent => { + schemas ??= {} as Schemas; const base = schemaComponent(DatabaseURN({ name: databaseName }), { migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...schemas], + components: [ + ...(migrationsOrComponents.components ?? []), + ...Object.values(schemas), + ], }); return { ...base, databaseName, get schemas() { - return mapSchemaComponentsOfType( - base.components, - DatabaseSchemaURNType, - (c) => c.schemaName, - ); + const schemasMap = + mapSchemaComponentsOfType( + base.components, + DatabaseSchemaURNType, + (c) => c.schemaName, + ); + + return Object.assign(schemasMap, schemas); }, addSchema: (schema: string | DatabaseSchemaSchemaComponent) => base.addComponent( diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts index 20bdafdc..a01dbeb5 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -7,6 +7,7 @@ import { import { TableURNType, tableSchemaComponent, + type AnyTableSchemaComponent, type TableSchemaComponent, } from './tableSchemaComponent'; @@ -21,40 +22,54 @@ export const DatabaseSchemaURN = ({ name: string; }): DatabaseSchemaURN => `${DatabaseSchemaURNType}:${name}`; -export type DatabaseSchemaSchemaComponent = SchemaComponent< +export type DatabaseSchemaTables< + Tables extends AnyTableSchemaComponent = AnyTableSchemaComponent, +> = Record; + +export type DatabaseSchemaSchemaComponent< + Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +> = SchemaComponent< DatabaseSchemaURN, Readonly<{ schemaName: string; - tables: ReadonlyMap; + tables: ReadonlyMap & Tables; addTable: (table: string | TableSchemaComponent) => TableSchemaComponent; }> >; -export const databaseSchemaSchemaComponent = ({ +export type AnyDatabaseSchemaSchemaComponent = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + DatabaseSchemaSchemaComponent; + +export const databaseSchemaSchemaComponent = < + Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +>({ schemaName, - tableNames, + tables, ...migrationsOrComponents }: { schemaName: string; - tableNames?: string[]; -} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { - const tables = - tableNames?.map((tableName) => tableSchemaComponent({ tableName })) ?? []; - + tables?: Tables; +} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { const base = schemaComponent(DatabaseSchemaURN({ name: schemaName }), { migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...tables], + components: [ + ...(migrationsOrComponents.components ?? []), + ...Object.values(tables ?? {}), + ], }); return { ...base, schemaName, get tables() { - return mapSchemaComponentsOfType( + const tablesMap = mapSchemaComponentsOfType( base.components, TableURNType, (c) => c.tableName, ); + + return Object.assign(tablesMap, tables); }, addTable: (table: string | TableSchemaComponent) => base.addComponent( diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 425066c4..813d1fb5 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -20,41 +20,54 @@ export const TableURNType: TableURNType = 'sc:dumbo:table'; export const TableURN = ({ name }: { name: string }): TableURN => `${TableURNType}:${name}`; -export type TableSchemaComponent = SchemaComponent< - TableURN, - Readonly<{ - tableName: string; - columns: ReadonlyMap; - indexes: ReadonlyMap; - addColumn: (column: ColumnSchemaComponent) => ColumnSchemaComponent; - addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; - }> ->; +export type TableColumns = Record; -export const tableSchemaComponent = ({ +export type TableSchemaComponent = + SchemaComponent< + TableURN, + Readonly<{ + tableName: string; + columns: ReadonlyMap & Columns; + indexes: ReadonlyMap; + addColumn: (column: ColumnSchemaComponent) => ColumnSchemaComponent; + addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; + }> + >; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyTableSchemaComponent = TableSchemaComponent; + +export const tableSchemaComponent = < + Columns extends TableColumns = TableColumns, +>({ tableName, columns, ...migrationsOrComponents }: { tableName: string; - columns?: ColumnSchemaComponent[]; -} & SchemaComponentOptions): TableSchemaComponent => { - columns ??= []; + columns?: Columns; +} & SchemaComponentOptions): TableSchemaComponent => { + columns ??= {} as Columns; const base = schemaComponent(TableURN({ name: tableName }), { migrations: migrationsOrComponents.migrations ?? [], - components: [...(migrationsOrComponents.components ?? []), ...columns], + components: [ + ...(migrationsOrComponents.components ?? []), + ...Object.values(columns), + ], }); return { ...base, tableName, get columns() { - return mapSchemaComponentsOfType( + const columnsMap = mapSchemaComponentsOfType( base.components, ColumnURNType, (c) => c.columnName, ); + + return Object.assign(columnsMap, columns); }, get indexes() { return mapSchemaComponentsOfType( diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 2f25c06b..81206610 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,13 +1,17 @@ import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; import { + type AnyDatabaseSchemaSchemaComponent, columnSchemaComponent, type ColumnSchemaComponent, databaseSchemaComponent, type DatabaseSchemaComponent, + type DatabaseSchemas, databaseSchemaSchemaComponent, type DatabaseSchemaSchemaComponent, + type DatabaseSchemaTables, indexSchemaComponent, type IndexSchemaComponent, + type TableColumns, tableSchemaComponent, type TableSchemaComponent, } from '../components'; @@ -44,51 +48,52 @@ const dumboIndex = ( ...options, }); -const dumboTable = ( +const dumboTable = ( name: string, definition: { - columns?: Record; + columns?: Columns; indexes?: Record; } & SchemaComponentOptions, -): TableSchemaComponent => { +): TableSchemaComponent => { const { columns, indexes, ...options } = definition; - const components = [ - ...(columns ? Object.values(columns) : []), - ...(indexes ? Object.values(indexes) : []), - ]; + const components = [...(indexes ? Object.values(indexes) : [])]; return tableSchemaComponent({ tableName: name, + columns: columns ?? ({} as Columns), components, ...options, }); }; -function dumboDatabaseSchema( - tables: Record, -): DatabaseSchemaSchemaComponent; -function dumboDatabaseSchema( +function dumboDatabaseSchema< + Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +>(tables: Tables): DatabaseSchemaSchemaComponent; +function dumboDatabaseSchema< + Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +>( schemaName: string, - tables: Record, + tables: Tables, options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent; -function dumboDatabaseSchema( - nameOrTables: string | Record, - tables?: Record, +): DatabaseSchemaSchemaComponent; +function dumboDatabaseSchema< + Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +>( + nameOrTables: string | Tables, + tables?: Tables, options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent { +): DatabaseSchemaSchemaComponent { const schemaName = typeof nameOrTables === 'string' ? nameOrTables : DEFAULT_DATABASE_SCHEMA_NAME; - const components = Object.values( - (typeof nameOrTables === 'string' ? tables : nameOrTables) ?? {}, - ); - + const tablesMap = + (typeof nameOrTables === 'string' ? tables : nameOrTables) ?? + ({} as Tables); return databaseSchemaSchemaComponent({ schemaName, - components, + tables: tablesMap, ...options, }); } @@ -110,33 +115,30 @@ dumboDatabaseSchema.from = ( : dumboDatabaseSchema(tables); }; -function dumboDatabase( - schemas: Record, -): DatabaseSchemaComponent; -function dumboDatabase( +function dumboDatabase( + schemas: Schemas, +): DatabaseSchemaComponent; +function dumboDatabase( schema: DatabaseSchemaSchemaComponent, -): DatabaseSchemaComponent; -function dumboDatabase( +): DatabaseSchemaComponent; +function dumboDatabase( databaseName: string, - schemas: Record, + schemas: Schemas, options?: SchemaComponentOptions, -): DatabaseSchemaComponent; -function dumboDatabase( +): DatabaseSchemaComponent; +function dumboDatabase( databaseName: string, - schema: DatabaseSchemaSchemaComponent, + schema: AnyDatabaseSchemaSchemaComponent, options?: SchemaComponentOptions, -): DatabaseSchemaComponent; -function dumboDatabase( - nameOrSchemas: - | string - | DatabaseSchemaSchemaComponent - | Record, +): DatabaseSchemaComponent; +function dumboDatabase( + nameOrSchemas: string | DatabaseSchemaSchemaComponent | Schemas, schemasOrOptions?: | DatabaseSchemaSchemaComponent - | Record + | Schemas | SchemaComponentOptions, options?: SchemaComponentOptions, -): DatabaseSchemaComponent { +): DatabaseSchemaComponent { const databaseName = typeof nameOrSchemas === 'string' ? nameOrSchemas : DEFAULT_DATABASE_NAME; @@ -161,30 +163,27 @@ function dumboDatabase( ? options : (schemasOrOptions as typeof options); - const schemaComponents: DatabaseSchemaSchemaComponent[] = []; - - for (const [_, schemaComponent] of Object.entries(schemaMap)) { - schemaComponents.push(schemaComponent); - } - return databaseSchemaComponent({ databaseName, - components: schemaComponents, + schemas: schemaMap as Schemas, ...dbOptions, }); } -dumboDatabase.from = ( +dumboDatabase.from = ( databaseName: string | undefined, schemaNames: string[], -): DatabaseSchemaComponent => { +): DatabaseSchemaComponent => { const schemas = schemaNames.reduce( (acc, schemaName) => { - acc[schemaName] = dumboDatabaseSchema(schemaName, {}); + acc[schemaName] = dumboDatabaseSchema( + schemaName, + {} as DatabaseSchemaTables, + ); return acc; }, {} as Record, - ); + ) as Schemas; return databaseName ? dumboDatabase(databaseName, schemas) diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 04a918fc..4a85aa8f 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -41,6 +41,8 @@ void describe('dumboSchema', () => { assert.ok(tbl.columns.has('id')); assert.ok(tbl.columns.has('email')); assert.ok(tbl.indexes.has('idx_email')); + assert.ok(tbl.columns.id !== undefined); + assert.ok(tbl.columns.email !== undefined); }); void it('should create a named schema', () => { @@ -55,6 +57,7 @@ void describe('dumboSchema', () => { assert.strictEqual(sch.schemaName, 'public'); assert.strictEqual(sch.tables.size, 1); assert.ok(sch.tables.has('users')); + assert.ok(sch.tables.users.columns.id !== undefined); }); void it('should create a default schema without name', () => { @@ -100,6 +103,9 @@ void describe('dumboSchema', () => { assert.strictEqual(db.databaseName, 'myapp'); assert.strictEqual(db.schemas.size, 1); assert.ok(db.schemas.has('public')); + assert.ok(db.schemas.public !== undefined); + assert.ok(db.schemas.public.tables.users !== undefined); + assert.ok(db.schemas.public.tables.users.columns.id !== undefined); }); void it('should handle DEFAULT_SCHEMA', () => { @@ -173,6 +179,6 @@ const multiSchemaDb = database('myapp', { }); // Access using name-based maps -const publicSchema = multiSchemaDb.schemas.get('public'); +const publicSchema = multiSchemaDb.schemas.public; const usersTable = publicSchema?.tables.get('users'); export const emailColumn = usersTable?.columns.get('email'); From 9cbd4dded9c8f5652c1e6057136bcb522127fe64 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Tue, 18 Nov 2025 18:41:38 +0100 Subject: [PATCH 28/39] Added columntype sql token to support typed query builders --- .../dumbo/src/core/sql/tokens/columnTokens.ts | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index 4d63d961..0c730f1f 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -2,6 +2,8 @@ import { SQLToken } from './sqlToken'; // TODO: Use URNs for sqltoken export type ColumnTypeToken< + // eslint-disable-next-line @typescript-eslint/no-unused-vars + ValueType, ColumnTypeName extends string = string, TProps extends Omit, 'sqlTokenType'> | undefined = | Omit, 'sqlTokenType'> @@ -9,32 +11,48 @@ export type ColumnTypeToken< > = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps>; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyColumnTypeToken = ColumnTypeToken; +export type AnyColumnTypeToken = ColumnTypeToken; -export type SerialToken = ColumnTypeToken<'SERIAL'>; +export type SerialToken = ColumnTypeToken; export const SerialToken = SQLToken('SQL_COLUMN_SERIAL'); -export type BigSerialToken = ColumnTypeToken<'BIGSERIAL'>; +export type BigSerialToken = ColumnTypeToken; export const BigSerialToken = SQLToken('SQL_COLUMN_BIGSERIAL'); -export type IntegerToken = ColumnTypeToken<'INTEGER'>; +export type IntegerToken = ColumnTypeToken; export const IntegerToken = SQLToken('SQL_COLUMN_INTEGER'); -export type BigIntegerToken = ColumnTypeToken<'BIGINT'>; +export type BigIntegerToken = ColumnTypeToken; export const BigIntegerToken = SQLToken('SQL_COLUMN_BIGINT'); -export type JSONBToken = ColumnTypeToken<'JSONB'>; -export const JSONBToken = SQLToken('SQL_COLUMN_JSONB'); +export type JSONBToken< + ValueType extends Record = Record, +> = ColumnTypeToken; + +export const JSONBToken = { + type: 'SQL_COLUMN_JSONB', + from: < + ValueType extends Record = Record, + >(): JSONBToken => { + return { + sqlTokenType: 'SQL_COLUMN_JSONB', + ['SQL_COLUMN_JSONB']: true, + } as unknown as JSONBToken; + }, + check: (token: unknown): token is JSONBToken => + SQLToken.check(token) && token.sqlTokenType === 'SQL_COLUMN_JSONB', +}; -export type TimestampToken = ColumnTypeToken<'TIMESTAMP'>; +export type TimestampToken = ColumnTypeToken; export const TimestampToken = SQLToken('SQL_COLUMN_TIMESTAMP'); -export type TimestamptzToken = ColumnTypeToken<'TIMESTAMPTZ'>; +export type TimestamptzToken = ColumnTypeToken; export const TimestamptzToken = SQLToken( 'SQL_COLUMN_TIMESTAMPTZ', ); export type VarcharToken = ColumnTypeToken< + string, 'VARCHAR', { length: number | 'max' } >; @@ -96,7 +114,7 @@ export const SQLColumnTypeTokensFactory = { BigInteger: BigIntegerToken.from(), BigSerial: BigSerialToken.from(), Integer: IntegerToken.from(), - JSONB: JSONBToken.from(), + JSONB: JSONBToken.from, Serial: SerialToken.from(), Timestamp: TimestampToken.from(), Timestamptz: TimestamptzToken.from(), From e9085d616b9189c0c33df371f397f4926594d5a8 Mon Sep 17 00:00:00 2001 From: oskardudycz Date: Tue, 18 Nov 2025 21:29:28 +0100 Subject: [PATCH 29/39] Added first draft of type interference for column and table types --- .../components/columnSchemaComponent.ts | 20 +- .../dumbo/src/core/schema/components/index.ts | 1 + .../schema/components/tableSchemaComponent.ts | 12 +- .../core/schema/dumboSchema/dumboSchema.ts | 12 +- .../dumbo/src/core/schema/typeInference.ts | 32 ++++ .../core/schema/typeInference.type.spec.ts | 180 ++++++++++++++++++ .../core/schema/typeInference.unit.spec.ts | 164 ++++++++++++++++ .../dumbo/src/core/sql/tokens/columnTokens.ts | 9 +- 8 files changed, 410 insertions(+), 20 deletions(-) create mode 100644 src/packages/dumbo/src/core/schema/typeInference.ts create mode 100644 src/packages/dumbo/src/core/schema/typeInference.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index 8402b8f9..68ff3cf4 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -1,4 +1,4 @@ -import type { SQLColumnToken } from '../../sql'; +import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; import { schemaComponent, type SchemaComponent, @@ -12,22 +12,32 @@ export const ColumnURNType: ColumnURNType = 'sc:dumbo:column'; export const ColumnURN = ({ name }: { name: string }): ColumnURN => `${ColumnURNType}:${name}`; -export type ColumnSchemaComponent = SchemaComponent< +export type ColumnSchemaComponent< + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +> = SchemaComponent< ColumnURN, Readonly<{ columnName: string; }> > & - SQLColumnToken; + SQLColumnToken; -export const columnSchemaComponent = ({ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyColumnSchemaComponent = ColumnSchemaComponent; + +export const columnSchemaComponent = < + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +>({ columnName, type, ...migrationsOrComponents }: { columnName: string; } & SchemaComponentOptions & - Omit): ColumnSchemaComponent => { + Omit< + SQLColumnToken, + 'name' | 'sqlTokenType' + >): ColumnSchemaComponent => { const sc = schemaComponent( ColumnURN({ name: columnName }), migrationsOrComponents, diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts index fe39d7a8..7bc1e10f 100644 --- a/src/packages/dumbo/src/core/schema/components/index.ts +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -9,6 +9,7 @@ export * from './databaseSchemaComponent'; export * from './databaseSchemaSchemaComponent'; export * from './indexSchemaComponent'; export * from './tableSchemaComponent'; +export * from '../typeInference'; export const schemaComponentURN = { database: DatabaseURN, diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 813d1fb5..0848dd25 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -6,7 +6,7 @@ import { } from '../schemaComponent'; import { ColumnURNType, - type ColumnSchemaComponent, + type AnyColumnSchemaComponent, } from './columnSchemaComponent'; import { IndexURNType, @@ -20,16 +20,16 @@ export const TableURNType: TableURNType = 'sc:dumbo:table'; export const TableURN = ({ name }: { name: string }): TableURN => `${TableURNType}:${name}`; -export type TableColumns = Record; +export type TableColumns = Record; export type TableSchemaComponent = SchemaComponent< TableURN, Readonly<{ tableName: string; - columns: ReadonlyMap & Columns; + columns: ReadonlyMap & Columns; indexes: ReadonlyMap; - addColumn: (column: ColumnSchemaComponent) => ColumnSchemaComponent; + addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; }> >; @@ -61,7 +61,7 @@ export const tableSchemaComponent = < ...base, tableName, get columns() { - const columnsMap = mapSchemaComponentsOfType( + const columnsMap = mapSchemaComponentsOfType( base.components, ColumnURNType, (c) => c.columnName, @@ -76,7 +76,7 @@ export const tableSchemaComponent = < (c) => c.indexName, ); }, - addColumn: (column: ColumnSchemaComponent) => base.addComponent(column), + addColumn: (column: AnyColumnSchemaComponent) => base.addComponent(column), addIndex: (index: IndexSchemaComponent) => base.addComponent(index), }; }; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 81206610..ff8490c7 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -24,13 +24,15 @@ import { const DEFAULT_DATABASE_NAME = '__default_database__'; const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; -const dumboColumn = ( +const dumboColumn = < + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +>( name: string, - type: string | AnyColumnTypeToken, + type: ColumnType, options: SchemaComponentOptions & - Omit = {}, -): ColumnSchemaComponent => - columnSchemaComponent({ + Omit, 'name' | 'type' | 'sqlTokenType'> = {}, +): ColumnSchemaComponent => + columnSchemaComponent({ columnName: name, type, ...options, diff --git a/src/packages/dumbo/src/core/schema/typeInference.ts b/src/packages/dumbo/src/core/schema/typeInference.ts new file mode 100644 index 00000000..55e55a46 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/typeInference.ts @@ -0,0 +1,32 @@ +import type { ColumnTypeToken } from '../sql/tokens/columnTokens'; +import type { ColumnSchemaComponent } from './components/columnSchemaComponent'; +import type { + AnyTableSchemaComponent, + TableColumns, + TableSchemaComponent, +} from './components/tableSchemaComponent'; + +export type InferColumnValueType = + ColumnType extends ColumnTypeToken< + infer ValueType, + infer _ColumnTypeName, + infer _TProps + > + ? ValueType + : ColumnType; + +export type InferColumnType = + T extends ColumnSchemaComponent + ? T extends { notNull: true } + ? InferColumnValueType + : InferColumnValueType | null + : unknown; + +export type InferTableRow = { + [K in keyof Columns]: InferColumnType; +}; + +export type InferTableType = + T extends TableSchemaComponent + ? InferTableRow + : never; diff --git a/src/packages/dumbo/src/core/schema/typeInference.type.spec.ts b/src/packages/dumbo/src/core/schema/typeInference.type.spec.ts new file mode 100644 index 00000000..6ff6a335 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/typeInference.type.spec.ts @@ -0,0 +1,180 @@ +import { SQL } from '../sql'; +import type { + BigIntegerToken, + BigSerialToken, + IntegerToken, + JSONBToken, + SerialToken, + TimestampToken, + TimestamptzToken, + VarcharToken, +} from '../sql/tokens/columnTokens'; +import { dumboSchema } from './dumboSchema'; +import type { + InferColumnType, + InferColumnValueType, + InferTableRow, + InferTableType, +} from './typeInference'; + +type Expect = T; +type Equal = + (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 + ? true + : false; + +const { table, column } = dumboSchema; +const { Serial, BigSerial, Integer, BigInteger, Varchar, Timestamp, JSONB } = + SQL.column.type; + +// InferColumnValueType - basic types +type _Test1 = Expect, number>>; +type _Test2 = Expect, bigint>>; +type _Test3 = Expect, number>>; +type _Test4 = Expect, bigint>>; +type _Test5 = Expect, string>>; +type _Test6 = Expect, Date>>; +type _Test7 = Expect, Date>>; + +// InferColumnValueType - JSONB with custom type +type CustomType = { foo: string; bar: number }; +type _Test8 = Expect< + Equal>, CustomType> +>; + +// InferColumnType - primary key is non-nullable +const _idColumn = column('id', Serial, { primaryKey: true }); +type _Test9 = Expect, number | null>>; + +// InferColumnType - notNull is non-nullable +const _emailColumn = column('email', Varchar(255), { notNull: true }); +type t = InferColumnType; +type _Test10 = Expect, string>>; + +// InferColumnType - default column is nullable +const _nicknameColumn = column('nickname', Varchar(100)); +type _Test11 = Expect< + Equal, string | null> +>; + +// InferColumnType - column with default is still nullable +const _createdAtColumn = column('createdAt', Timestamp, { + default: 'NOW()' as unknown as Date, +}); +type _Test12 = Expect< + Equal, Date | null> +>; + +// InferColumnType - unique column is nullable +const _usernameColumn = column('username', Varchar(50), { unique: true }); +type _Test13 = Expect< + Equal, string | null> +>; + +// InferColumnType - serial without primary key is nullable +const _sortOrderColumn = column('sortOrder', Serial); +type _Test14 = Expect< + Equal, number | null> +>; + +// InferColumnType - bigint types +const _bigIdColumn = column('bigId', BigSerial, { primaryKey: true }); +const _nullableBigIntColumn = column('bigValue', BigInteger); +type _Test15 = Expect, bigint>>; +type _Test16 = Expect< + Equal, bigint | null> +>; + +// InferTableRow - complex table with mixed nullability +const _usersTable = table('users', { + columns: { + id: column('id', Serial, { primaryKey: true }), + email: column('email', Varchar(255), { notNull: true }), + nickname: column('nickname', Varchar(100)), + age: column('age', Integer), + createdAt: column('createdAt', Timestamp, { default: 'NOW()' }), + username: column('username', Varchar(50), { unique: true }), + }, +}); +type UserRow = InferTableRow; +type _Test17 = Expect< + Equal< + UserRow, + { + id: number; + email: string; + nickname: string | null; + age: number | null; + createdAt: Date | null; + username: string | null; + } + > +>; + +// InferTableType - infer from TableSchemaComponent +const _productsTable = table('products', { + columns: { + id: column('id', BigSerial, { primaryKey: true }), + name: column('name', Varchar(255), { notNull: true }), + description: column('description', Varchar('max')), + price: column('price', Integer, { notNull: true }), + metadata: column('metadata', JSONB<{ tags: string[] }>()), + }, +}); +type ProductRow = InferTableType; +type _Test18 = Expect< + Equal< + ProductRow, + { + id: bigint; + name: string; + description: string | null; + price: number; + metadata: { tags: string[] } | null; + } + > +>; + +// InferTableType - table with all non-nullable columns +const _strictTable = table('strict', { + columns: { + id: column('id', Serial, { primaryKey: true }), + field1: column('field1', Varchar(100), { notNull: true }), + field2: column('field2', Integer, { notNull: true }), + field3: column('field3', Timestamp, { notNull: true }), + }, +}); +type StrictRow = InferTableType; +type _Test19 = Expect< + Equal< + StrictRow, + { + id: number; + field1: string; + field2: number; + field3: Date; + } + > +>; + +// InferTableType - table with all nullable columns (except PK) +const _nullableTable = table('nullable', { + columns: { + id: column('id', Serial, { primaryKey: true }), + field1: column('field1', Varchar(100)), + field2: column('field2', Integer), + field3: column('field3', Timestamp), + }, +}); +type NullableRow = InferTableType; +type _Test20 = Expect< + Equal< + NullableRow, + { + id: number; + field1: string | null; + field2: number | null; + field3: Date | null; + } + > +>; diff --git a/src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts b/src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts new file mode 100644 index 00000000..ce9c9360 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts @@ -0,0 +1,164 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; +import { SQL } from '../sql'; +import { dumboSchema } from './dumboSchema'; +import type { InferTableType } from './typeInference'; + +const { table, column } = dumboSchema; +const { Serial, Varchar, Integer, Timestamp, JSONB } = SQL.column.type; + +void describe('Type Inference Runtime Tests', () => { + void it('should compile successfully with basic table', () => { + const _users = table('users', { + columns: { + id: column('id', Serial, { primaryKey: true }), + email: column('email', Varchar(255), { notNull: true }), + nickname: column('nickname', Varchar(100)), + }, + }); + + type UserRow = InferTableType; + + const sampleUser: UserRow = { + id: 1, + email: 'test@example.com', + nickname: 'tester', + }; + + assert.strictEqual(sampleUser.id, 1); + assert.strictEqual(sampleUser.email, 'test@example.com'); + assert.strictEqual(sampleUser.nickname, 'tester'); + }); + + void it('should allow null for nullable columns', () => { + const _users2 = table('users', { + columns: { + id: column('id', Serial, { primaryKey: true }), + nickname: column('nickname', Varchar(100)), + }, + }); + + type UserRow = InferTableType; + + const user1: UserRow = { + id: 1, + nickname: null, + }; + + const user2: UserRow = { + id: 2, + nickname: 'test', + }; + + assert.strictEqual(user1.nickname, null); + assert.strictEqual(user2.nickname, 'test'); + }); + + void it('should work with JSONB custom types', () => { + const _products = table('products', { + columns: { + id: column('id', Serial, { primaryKey: true }), + metadata: column('metadata', JSONB<{ tags: string[] }>()), + }, + }); + + type ProductRow = InferTableType; + + const product: ProductRow = { + id: 1, + metadata: { tags: ['electronics', 'sale'] }, + }; + + assert.strictEqual(product.id, 1); + assert.deepStrictEqual(product.metadata, { tags: ['electronics', 'sale'] }); + }); + + void it('should work with mixed nullable and non-nullable columns', () => { + const _posts = table('posts', { + columns: { + id: column('id', Serial, { primaryKey: true }), + title: column('title', Varchar(255), { notNull: true }), + content: column('content', Varchar('max'), { notNull: true }), + publishedAt: column('publishedAt', Timestamp), + viewCount: column('viewCount', Integer), + }, + }); + + type PostRow = InferTableType; + + const draftPost: PostRow = { + id: 1, + title: 'My First Post', + content: 'This is the content', + publishedAt: null, + viewCount: null, + }; + + const publishedPost: PostRow = { + id: 2, + title: 'Published Post', + content: 'Published content', + publishedAt: new Date(), + viewCount: 42, + }; + + assert.strictEqual(draftPost.publishedAt, null); + assert.strictEqual(draftPost.viewCount, null); + assert.ok(publishedPost.publishedAt instanceof Date); + assert.strictEqual(publishedPost.viewCount, 42); + }); + + void it('should work with default values (still nullable)', () => { + const _events = table('events', { + columns: { + id: column('id', Serial, { primaryKey: true }), + createdAt: column('createdAt', Timestamp, { default: 'NOW()' }), + }, + }); + + type EventRow = InferTableType; + + const event1: EventRow = { + id: 1, + createdAt: new Date(), + }; + + const event2: EventRow = { + id: 2, + createdAt: null, + }; + + assert.ok(event1.createdAt instanceof Date); + assert.strictEqual(event2.createdAt, null); + }); + + void it('example: type-safe query result processing', () => { + const _users3 = table('users', { + columns: { + id: column('id', Serial, { primaryKey: true }), + email: column('email', Varchar(255), { notNull: true }), + nickname: column('nickname', Varchar(100)), + age: column('age', Integer), + }, + }); + + type UserRow = InferTableType; + + const mockQueryResults: UserRow[] = [ + { id: 1, email: 'alice@example.com', nickname: 'alice', age: 30 }, + { id: 2, email: 'bob@example.com', nickname: null, age: null }, + { id: 3, email: 'charlie@example.com', nickname: 'charlie', age: 25 }, + ]; + + const processedResults = mockQueryResults.map((user) => ({ + id: user.id, + email: (user.email as string).toUpperCase(), + displayName: user.nickname ?? 'Anonymous', + isAdult: user.age !== null && user.age >= 18, + })); + + assert.strictEqual(processedResults[0]?.email, 'ALICE@EXAMPLE.COM'); + assert.strictEqual(processedResults[1]?.displayName, 'Anonymous'); + assert.strictEqual(processedResults[2]?.isAdult, true); + }); +}); diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index 0c730f1f..a9376261 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -2,13 +2,12 @@ import { SQLToken } from './sqlToken'; // TODO: Use URNs for sqltoken export type ColumnTypeToken< - // eslint-disable-next-line @typescript-eslint/no-unused-vars ValueType, ColumnTypeName extends string = string, TProps extends Omit, 'sqlTokenType'> | undefined = | Omit, 'sqlTokenType'> | undefined, -> = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps>; +> = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps> & { __brand: ValueType }; // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyColumnTypeToken = ColumnTypeToken; @@ -63,11 +62,13 @@ export const VarcharToken = SQLToken( }), ); -export type SQLColumnToken = SQLToken< +export type SQLColumnToken< + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +> = SQLToken< 'SQL_COLUMN', { name: string; - type: ColumnType | AnyColumnTypeToken; + type: ColumnType; notNull?: boolean; unique?: boolean; primaryKey?: boolean; From a61bd24d791d26ffb3dc2579bcd3840b20586544 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 10:28:28 +0100 Subject: [PATCH 30/39] Added explicit json type mapping --- .../dumbo/src/core/schema/typeInference.ts | 12 +- .../dumbo/src/core/sql/tokens/columnTokens.ts | 225 +++++++++++++++--- 2 files changed, 199 insertions(+), 38 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/typeInference.ts b/src/packages/dumbo/src/core/schema/typeInference.ts index 55e55a46..356cf631 100644 --- a/src/packages/dumbo/src/core/schema/typeInference.ts +++ b/src/packages/dumbo/src/core/schema/typeInference.ts @@ -1,5 +1,8 @@ import type { ColumnTypeToken } from '../sql/tokens/columnTokens'; -import type { ColumnSchemaComponent } from './components/columnSchemaComponent'; +import type { + AnyColumnSchemaComponent, + ColumnSchemaComponent, +} from './components/columnSchemaComponent'; import type { AnyTableSchemaComponent, TableColumns, @@ -8,14 +11,15 @@ import type { export type InferColumnValueType = ColumnType extends ColumnTypeToken< - infer ValueType, + infer _JSType, infer _ColumnTypeName, - infer _TProps + infer _TProps, + infer ValueType > ? ValueType : ColumnType; -export type InferColumnType = +export type InferColumnType = T extends ColumnSchemaComponent ? T extends { notNull: true } ? InferColumnValueType diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index a9376261..5fc89df4 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -1,32 +1,165 @@ import { SQLToken } from './sqlToken'; +export type JSONValueType = + | Record + | Array + | string + | number + | boolean + | null; + +export type JSONValueTypeName = + | 'value_type:json:object' + | 'value_type:json:array' + | 'value_type:json:string' + | 'value_type:json:number' + | 'value_type:json:boolean' + | 'value_type:json:null'; + +export type JavaScriptValueType = + | Record + | Array + | string + | number + | boolean + | null + | undefined + | Date + | bigint; + +export type JavaScriptValueTypeName = + | 'value_type:js:object' + | 'value_type:js:array' + | 'value_type:js:string' + | 'value_type:js:number' + | 'value_type:js:boolean' + | 'value_type:js:null' + | 'value_type:js:undefined' + | 'value_type:js:date' + | 'value_type:js:bigint'; + +export type JavaScriptValueTypeToNameMap = { + [K in JavaScriptValueType as K extends Record + ? 'value_type:js:object' + : K extends Array + ? 'value_type:js:array' + : K extends string + ? 'value_type:js:string' + : K extends number + ? 'value_type:js:number' + : K extends boolean + ? 'value_type:js:boolean' + : K extends null + ? 'value_type:js:null' + : K extends undefined + ? 'value_type:js:undefined' + : K extends Date + ? 'value_type:js:date' + : K extends bigint + ? 'value_type:js:bigint' + : never]: K; +}; + // TODO: Use URNs for sqltoken export type ColumnTypeToken< - ValueType, + JSValueTypeName extends JavaScriptValueTypeName = JavaScriptValueTypeName, ColumnTypeName extends string = string, TProps extends Omit, 'sqlTokenType'> | undefined = | Omit, 'sqlTokenType'> | undefined, -> = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps> & { __brand: ValueType }; + ValueType = undefined, +> = SQLToken<`SQL_COLUMN_${ColumnTypeName}`, TProps> & { + __brand: ValueType extends undefined + ? JavaScriptValueTypeToNameMap[JSValueTypeName] + : ValueType; + jsTypeName: JSValueTypeName; +}; + +export const ColumnTypeToken = < + SQLTokenType extends AnyColumnTypeToken, + TInput = keyof Omit< + SQLTokenType, + 'sqlTokenType' | '__brand' | 'jsTypeName' + > extends never + ? void + : Omit, +>( + sqlTokenType: SQLTokenType['sqlTokenType'], + jsTypeName: SQLTokenType['jsTypeName'], + map?: ( + input: TInput, + ) => Omit, +) => { + const factory = (input: TInput): SQLTokenType => { + let props: Omit; + + if (map !== undefined) { + props = map(input) as SQLTokenType; + } else if (input === undefined || input === null) { + props = {} as Omit< + SQLTokenType, + 'sqlTokenType' | '__brand' | 'jsTypeName' + >; + } else if (typeof input === 'object' && !Array.isArray(input)) { + // If input is already an object (but not array), spread it + props = input as Omit< + SQLTokenType, + 'sqlTokenType' | '__brand' | 'jsTypeName' + >; + } else { + throw new Error( + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + `Cannot create SQLToken of type ${sqlTokenType} with input: ${input}`, + ); + } + + return { + sqlTokenType: sqlTokenType, + [sqlTokenType]: true, + jsTypeName, + ...props, + } as unknown as SQLTokenType; + }; + + const check = (token: unknown): token is SQLTokenType => + SQLToken.check(token) && token.sqlTokenType === sqlTokenType; + + return { from: factory, check: check, type: sqlTokenType }; +}; // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyColumnTypeToken = ColumnTypeToken; -export type SerialToken = ColumnTypeToken; -export const SerialToken = SQLToken('SQL_COLUMN_SERIAL'); +export type SerialToken = ColumnTypeToken<'value_type:js:number', 'SERIAL'>; +export const SerialToken = ColumnTypeToken( + 'SQL_COLUMN_SERIAL', + 'value_type:js:number', +); -export type BigSerialToken = ColumnTypeToken; -export const BigSerialToken = SQLToken('SQL_COLUMN_BIGSERIAL'); +export type BigSerialToken = ColumnTypeToken< + 'value_type:js:bigint', + 'BIGSERIAL' +>; +export const BigSerialToken = ColumnTypeToken( + 'SQL_COLUMN_BIGSERIAL', + 'value_type:js:bigint', +); -export type IntegerToken = ColumnTypeToken; -export const IntegerToken = SQLToken('SQL_COLUMN_INTEGER'); +export type IntegerToken = ColumnTypeToken<'value_type:js:number', 'INTEGER'>; +export const IntegerToken = ColumnTypeToken( + 'SQL_COLUMN_INTEGER', + 'value_type:js:number', +); -export type BigIntegerToken = ColumnTypeToken; -export const BigIntegerToken = SQLToken('SQL_COLUMN_BIGINT'); +export type BigIntegerToken = ColumnTypeToken<'value_type:js:bigint', 'BIGINT'>; +export const BigIntegerToken = ColumnTypeToken( + 'SQL_COLUMN_BIGINT', + 'value_type:js:bigint', +); export type JSONBToken< ValueType extends Record = Record, -> = ColumnTypeToken; +> = ColumnTypeToken<'value_type:js:object', 'JSONB', undefined, ValueType>; export const JSONBToken = { type: 'SQL_COLUMN_JSONB', @@ -36,55 +169,79 @@ export const JSONBToken = { return { sqlTokenType: 'SQL_COLUMN_JSONB', ['SQL_COLUMN_JSONB']: true, - } as unknown as JSONBToken; + } as unknown as JSONBToken; }, - check: (token: unknown): token is JSONBToken => + check: = Record>( + token: unknown, + ): token is JSONBToken => SQLToken.check(token) && token.sqlTokenType === 'SQL_COLUMN_JSONB', }; -export type TimestampToken = ColumnTypeToken; -export const TimestampToken = SQLToken('SQL_COLUMN_TIMESTAMP'); +export type TimestampToken = ColumnTypeToken<'value_type:js:date', 'TIMESTAMP'>; +export const TimestampToken = ColumnTypeToken( + 'SQL_COLUMN_TIMESTAMP', + 'value_type:js:date', +); -export type TimestamptzToken = ColumnTypeToken; -export const TimestamptzToken = SQLToken( +export type TimestamptzToken = ColumnTypeToken< + 'value_type:js:date', + 'TIMESTAMPTZ' +>; +export const TimestamptzToken = ColumnTypeToken( 'SQL_COLUMN_TIMESTAMPTZ', + 'value_type:js:date', ); export type VarcharToken = ColumnTypeToken< - string, + 'value_type:js:string', 'VARCHAR', { length: number | 'max' } >; -export const VarcharToken = SQLToken( +export const VarcharToken = ColumnTypeToken( 'SQL_COLUMN_VARCHAR', - (length?: number | 'max') => ({ - length: length ?? 'max', - }), + 'value_type:js:string', + (length?: number | 'max') => + ({ + length: length ?? 'max', + jsTypeName: 'value_type:js:string', + }) as Omit, ); export type SQLColumnToken< ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, > = SQLToken< 'SQL_COLUMN', - { - name: string; - type: ColumnType; - notNull?: boolean; - unique?: boolean; - primaryKey?: boolean; - default?: ColumnType | SQLToken; - } + | { + name: string; + type: ColumnType; + notNull: true; + unique?: boolean; + primaryKey?: boolean; + default?: ColumnType | SQLToken; + } + | { + name: string; + type: ColumnType; + notNull?: false; + unique?: boolean; + default?: ColumnType | SQLToken; + } >; -export type AutoIncrementSQLColumnToken = SQLToken< - 'SQL_COLUMN_AUTO_INCREMENT', +export type AutoIncrementSQLColumnToken = ColumnTypeToken< + 'value_type:js:bigint', + 'AUTO_INCREMENT', { primaryKey: boolean; bigint?: boolean; - } + }, + bigint >; export const AutoIncrementSQLColumnToken = - SQLToken('SQL_COLUMN_AUTO_INCREMENT'); + ColumnTypeToken( + 'SQL_COLUMN_AUTO_INCREMENT', + 'value_type:js:bigint', + ); export const SQLColumnTypeTokens = { AutoIncrement: AutoIncrementSQLColumnToken, From 305d825f8f5dae764b0b425742cba546114a0866 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 11:00:29 +0100 Subject: [PATCH 31/39] Fixed nullability mapping of table columns --- .../components/columnSchemaComponent.ts | 54 +++++++++++++------ .../dumbo/src/core/schema/components/index.ts | 2 +- .../schema/{ => components}/typeInference.ts | 8 +-- .../typeInference.type.spec.ts | 30 +++++------ .../typeInference.unit.spec.ts | 10 ++-- .../core/schema/dumboSchema/dumboSchema.ts | 18 ++++--- .../dumbo/src/core/sql/tokens/columnTokens.ts | 26 +++++++-- 7 files changed, 97 insertions(+), 51 deletions(-) rename src/packages/dumbo/src/core/schema/{ => components}/typeInference.ts (81%) rename src/packages/dumbo/src/core/schema/{ => components}/typeInference.type.spec.ts (88%) rename src/packages/dumbo/src/core/schema/{ => components}/typeInference.unit.spec.ts (95%) diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index 68ff3cf4..1f7ca340 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -25,29 +25,49 @@ export type ColumnSchemaComponent< // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyColumnSchemaComponent = ColumnSchemaComponent; +export type ColumnSchemaComponentOptions< + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +> = Omit, 'name' | 'sqlTokenType'> & + SchemaComponentOptions; + export const columnSchemaComponent = < ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, ->({ - columnName, - type, - ...migrationsOrComponents -}: { - columnName: string; -} & SchemaComponentOptions & - Omit< - SQLColumnToken, - 'name' | 'sqlTokenType' - >): ColumnSchemaComponent => { - const sc = schemaComponent( - ColumnURN({ name: columnName }), - migrationsOrComponents, - ); - - return { + TOptions extends + ColumnSchemaComponentOptions = ColumnSchemaComponentOptions, +>( + params: { + columnName: string; + } & TOptions, +): ColumnSchemaComponent & + (TOptions extends { notNull: true } | { primaryKey: true } + ? { notNull: true } + : { notNull?: false }) => { + const { + columnName, + type, + notNull, + unique, + primaryKey, + default: defaultValue, + ...schemaOptions + } = params; + + const sc = schemaComponent(ColumnURN({ name: columnName }), schemaOptions); + + const result: Record = { ...sc, columnName, + notNull, + unique, + primaryKey, + defaultValue, sqlTokenType: 'SQL_COLUMN', name: columnName, type, }; + + return result as ColumnSchemaComponent & + (TOptions extends { notNull: true } | { primaryKey: true } + ? { notNull: true } + : { notNull?: false }); }; diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts index 7bc1e10f..fd674eb6 100644 --- a/src/packages/dumbo/src/core/schema/components/index.ts +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -9,7 +9,7 @@ export * from './databaseSchemaComponent'; export * from './databaseSchemaSchemaComponent'; export * from './indexSchemaComponent'; export * from './tableSchemaComponent'; -export * from '../typeInference'; +export * from './typeInference'; export const schemaComponentURN = { database: DatabaseURN, diff --git a/src/packages/dumbo/src/core/schema/typeInference.ts b/src/packages/dumbo/src/core/schema/components/typeInference.ts similarity index 81% rename from src/packages/dumbo/src/core/schema/typeInference.ts rename to src/packages/dumbo/src/core/schema/components/typeInference.ts index 356cf631..b8928ae2 100644 --- a/src/packages/dumbo/src/core/schema/typeInference.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.ts @@ -1,13 +1,13 @@ -import type { ColumnTypeToken } from '../sql/tokens/columnTokens'; +import type { ColumnTypeToken } from '../../sql/tokens/columnTokens'; import type { AnyColumnSchemaComponent, ColumnSchemaComponent, -} from './components/columnSchemaComponent'; +} from './columnSchemaComponent'; import type { AnyTableSchemaComponent, TableColumns, TableSchemaComponent, -} from './components/tableSchemaComponent'; +} from './tableSchemaComponent'; export type InferColumnValueType = ColumnType extends ColumnTypeToken< @@ -21,7 +21,7 @@ export type InferColumnValueType = export type InferColumnType = T extends ColumnSchemaComponent - ? T extends { notNull: true } + ? T extends { notNull: true } | { primaryKey: true } ? InferColumnValueType : InferColumnValueType | null : unknown; diff --git a/src/packages/dumbo/src/core/schema/typeInference.type.spec.ts b/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts similarity index 88% rename from src/packages/dumbo/src/core/schema/typeInference.type.spec.ts rename to src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts index 6ff6a335..2f5643f7 100644 --- a/src/packages/dumbo/src/core/schema/typeInference.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts @@ -1,4 +1,4 @@ -import { SQL } from '../sql'; +import { SQL } from '../../sql'; import type { BigIntegerToken, BigSerialToken, @@ -8,8 +8,8 @@ import type { TimestampToken, TimestamptzToken, VarcharToken, -} from '../sql/tokens/columnTokens'; -import { dumboSchema } from './dumboSchema'; +} from '../../sql/tokens/columnTokens'; +import { dumboSchema } from '../dumboSchema'; import type { InferColumnType, InferColumnValueType, @@ -44,11 +44,10 @@ type _Test8 = Expect< // InferColumnType - primary key is non-nullable const _idColumn = column('id', Serial, { primaryKey: true }); -type _Test9 = Expect, number | null>>; +type _Test9 = Expect, number>>; // InferColumnType - notNull is non-nullable const _emailColumn = column('email', Varchar(255), { notNull: true }); -type t = InferColumnType; type _Test10 = Expect, string>>; // InferColumnType - default column is nullable @@ -59,7 +58,7 @@ type _Test11 = Expect< // InferColumnType - column with default is still nullable const _createdAtColumn = column('createdAt', Timestamp, { - default: 'NOW()' as unknown as Date, + default: SQL.plain(`NOW()`), }); type _Test12 = Expect< Equal, Date | null> @@ -86,17 +85,18 @@ type _Test16 = Expect< >; // InferTableRow - complex table with mixed nullability +const _usersColumns = { + id: column('id', Serial, { primaryKey: true }), + email: column('email', Varchar(255), { notNull: true }), + nickname: column('nickname', Varchar(100)), + age: column('age', Integer), + createdAt: column('createdAt', Timestamp, { default: SQL.plain(`NOW()`) }), + username: column('username', Varchar(50), { unique: true }), +}; const _usersTable = table('users', { - columns: { - id: column('id', Serial, { primaryKey: true }), - email: column('email', Varchar(255), { notNull: true }), - nickname: column('nickname', Varchar(100)), - age: column('age', Integer), - createdAt: column('createdAt', Timestamp, { default: 'NOW()' }), - username: column('username', Varchar(50), { unique: true }), - }, + columns: _usersColumns, }); -type UserRow = InferTableRow; +type UserRow = InferTableRow; type _Test17 = Expect< Equal< UserRow, diff --git a/src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts b/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts similarity index 95% rename from src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts rename to src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts index ce9c9360..8f5daf3d 100644 --- a/src/packages/dumbo/src/core/schema/typeInference.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { SQL } from '../sql'; -import { dumboSchema } from './dumboSchema'; +import { SQL } from '../../sql'; +import { dumboSchema } from '../dumboSchema'; import type { InferTableType } from './typeInference'; const { table, column } = dumboSchema; @@ -112,7 +112,9 @@ void describe('Type Inference Runtime Tests', () => { const _events = table('events', { columns: { id: column('id', Serial, { primaryKey: true }), - createdAt: column('createdAt', Timestamp, { default: 'NOW()' }), + createdAt: column('createdAt', Timestamp, { + default: SQL.plain(`NOW()`), + }), }, }); @@ -152,7 +154,7 @@ void describe('Type Inference Runtime Tests', () => { const processedResults = mockQueryResults.map((user) => ({ id: user.id, - email: (user.email as string).toUpperCase(), + email: user.email.toUpperCase(), displayName: user.nickname ?? 'Anonymous', isAdult: user.age !== null && user.age >= 18, })); diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index ff8490c7..61c203de 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -2,7 +2,7 @@ import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; import { type AnyDatabaseSchemaSchemaComponent, columnSchemaComponent, - type ColumnSchemaComponent, + type ColumnSchemaComponentOptions, databaseSchemaComponent, type DatabaseSchemaComponent, type DatabaseSchemas, @@ -26,17 +26,23 @@ const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = < ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, + TOptions extends SchemaComponentOptions & + Omit, 'name' | 'type' | 'sqlTokenType'> = Omit< + ColumnSchemaComponentOptions, + 'type' + >, >( name: string, type: ColumnType, - options: SchemaComponentOptions & - Omit, 'name' | 'type' | 'sqlTokenType'> = {}, -): ColumnSchemaComponent => - columnSchemaComponent({ + options?: TOptions, +): ReturnType< + typeof columnSchemaComponent +> => + columnSchemaComponent({ columnName: name, type, ...options, - }); + } as { columnName: string } & TOptions & { type: ColumnType }); const dumboIndex = ( name: string, diff --git a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts index 5fc89df4..bb55b61f 100644 --- a/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts +++ b/src/packages/dumbo/src/core/sql/tokens/columnTokens.ts @@ -207,10 +207,9 @@ export const VarcharToken = ColumnTypeToken( }) as Omit, ); -export type SQLColumnToken< +export type NotNullableSQLColumnTokenProps< ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, -> = SQLToken< - 'SQL_COLUMN', +> = | { name: string; type: ColumnType; @@ -224,8 +223,27 @@ export type SQLColumnToken< type: ColumnType; notNull?: false; unique?: boolean; + primaryKey: never; default?: ColumnType | SQLToken; - } + }; + +export type NullableSQLColumnTokenProps< + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +> = { + name: string; + type: ColumnType; + notNull?: false; + unique?: boolean; + primaryKey?: false; + default?: ColumnType | SQLToken; +}; + +export type SQLColumnToken< + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, +> = SQLToken< + 'SQL_COLUMN', + | NotNullableSQLColumnTokenProps + | NullableSQLColumnTokenProps >; export type AutoIncrementSQLColumnToken = ColumnTypeToken< From aaae02fd27157b8b6f8f80be993ed38fe717aabc Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 12:07:19 +0100 Subject: [PATCH 32/39] Adjusted samples for types interference --- src/packages/dumbo/src/core/index.ts | 1 + .../core/schema/components/typeInference.ts | 12 ++--- .../components/typeInference.type.spec.ts | 49 +++++++++---------- .../components/typeInference.unit.spec.ts | 14 +++--- .../dumboSchema/dumboSchema.unit.spec.ts | 20 +++++--- src/packages/dumbo/src/core/testing/index.ts | 1 + .../dumbo/src/core/testing/typesTesting.ts | 5 ++ 7 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 src/packages/dumbo/src/core/testing/index.ts create mode 100644 src/packages/dumbo/src/core/testing/typesTesting.ts diff --git a/src/packages/dumbo/src/core/index.ts b/src/packages/dumbo/src/core/index.ts index 5ea06e77..dfba1a8e 100644 --- a/src/packages/dumbo/src/core/index.ts +++ b/src/packages/dumbo/src/core/index.ts @@ -17,6 +17,7 @@ export * from './query'; export * from './schema'; export * from './serializer'; export * from './sql'; +export * from './testing'; export * from './tracing'; export type Dumbo< diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.ts b/src/packages/dumbo/src/core/schema/components/typeInference.ts index b8928ae2..9afc9710 100644 --- a/src/packages/dumbo/src/core/schema/components/typeInference.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.ts @@ -9,7 +9,7 @@ import type { TableSchemaComponent, } from './tableSchemaComponent'; -export type InferColumnValueType = +export type InferColumnType = ColumnType extends ColumnTypeToken< infer _JSType, infer _ColumnTypeName, @@ -19,18 +19,18 @@ export type InferColumnValueType = ? ValueType : ColumnType; -export type InferColumnType = +export type TableColumnType = T extends ColumnSchemaComponent ? T extends { notNull: true } | { primaryKey: true } - ? InferColumnValueType - : InferColumnValueType | null + ? InferColumnType + : InferColumnType | null : unknown; export type InferTableRow = { - [K in keyof Columns]: InferColumnType; + [K in keyof Columns]: TableColumnType; }; -export type InferTableType = +export type TableRowType = T extends TableSchemaComponent ? InferTableRow : never; diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts b/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts index 2f5643f7..d95c15ea 100644 --- a/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts @@ -9,51 +9,46 @@ import type { TimestamptzToken, VarcharToken, } from '../../sql/tokens/columnTokens'; +import type { Expect, Equal } from '../../testing'; import { dumboSchema } from '../dumboSchema'; import type { InferColumnType, - InferColumnValueType, InferTableRow, - InferTableType, + TableColumnType, + TableRowType, } from './typeInference'; -type Expect = T; -type Equal = - (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 - ? true - : false; - const { table, column } = dumboSchema; const { Serial, BigSerial, Integer, BigInteger, Varchar, Timestamp, JSONB } = SQL.column.type; // InferColumnValueType - basic types -type _Test1 = Expect, number>>; -type _Test2 = Expect, bigint>>; -type _Test3 = Expect, number>>; -type _Test4 = Expect, bigint>>; -type _Test5 = Expect, string>>; -type _Test6 = Expect, Date>>; -type _Test7 = Expect, Date>>; +type _Test1 = Expect, number>>; +type _Test2 = Expect, bigint>>; +type _Test3 = Expect, number>>; +type _Test4 = Expect, bigint>>; +type _Test5 = Expect, string>>; +type _Test6 = Expect, Date>>; +type _Test7 = Expect, Date>>; // InferColumnValueType - JSONB with custom type type CustomType = { foo: string; bar: number }; type _Test8 = Expect< - Equal>, CustomType> + Equal>, CustomType> >; // InferColumnType - primary key is non-nullable const _idColumn = column('id', Serial, { primaryKey: true }); -type _Test9 = Expect, number>>; +type _Test9 = Expect, number>>; // InferColumnType - notNull is non-nullable const _emailColumn = column('email', Varchar(255), { notNull: true }); -type _Test10 = Expect, string>>; +type _Test10 = Expect, string>>; // InferColumnType - default column is nullable const _nicknameColumn = column('nickname', Varchar(100)); type _Test11 = Expect< - Equal, string | null> + Equal, string | null> >; // InferColumnType - column with default is still nullable @@ -61,27 +56,27 @@ const _createdAtColumn = column('createdAt', Timestamp, { default: SQL.plain(`NOW()`), }); type _Test12 = Expect< - Equal, Date | null> + Equal, Date | null> >; // InferColumnType - unique column is nullable const _usernameColumn = column('username', Varchar(50), { unique: true }); type _Test13 = Expect< - Equal, string | null> + Equal, string | null> >; // InferColumnType - serial without primary key is nullable const _sortOrderColumn = column('sortOrder', Serial); type _Test14 = Expect< - Equal, number | null> + Equal, number | null> >; // InferColumnType - bigint types const _bigIdColumn = column('bigId', BigSerial, { primaryKey: true }); const _nullableBigIntColumn = column('bigValue', BigInteger); -type _Test15 = Expect, bigint>>; +type _Test15 = Expect, bigint>>; type _Test16 = Expect< - Equal, bigint | null> + Equal, bigint | null> >; // InferTableRow - complex table with mixed nullability @@ -121,7 +116,7 @@ const _productsTable = table('products', { metadata: column('metadata', JSONB<{ tags: string[] }>()), }, }); -type ProductRow = InferTableType; +type ProductRow = TableRowType; type _Test18 = Expect< Equal< ProductRow, @@ -144,7 +139,7 @@ const _strictTable = table('strict', { field3: column('field3', Timestamp, { notNull: true }), }, }); -type StrictRow = InferTableType; +type StrictRow = TableRowType; type _Test19 = Expect< Equal< StrictRow, @@ -166,7 +161,7 @@ const _nullableTable = table('nullable', { field3: column('field3', Timestamp), }, }); -type NullableRow = InferTableType; +type NullableRow = TableRowType; type _Test20 = Expect< Equal< NullableRow, diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts b/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts index 8f5daf3d..fbf2dfd3 100644 --- a/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; import { dumboSchema } from '../dumboSchema'; -import type { InferTableType } from './typeInference'; +import type { TableRowType } from './typeInference'; const { table, column } = dumboSchema; const { Serial, Varchar, Integer, Timestamp, JSONB } = SQL.column.type; @@ -17,7 +17,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type UserRow = InferTableType; + type UserRow = TableRowType; const sampleUser: UserRow = { id: 1, @@ -38,7 +38,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type UserRow = InferTableType; + type UserRow = TableRowType; const user1: UserRow = { id: 1, @@ -62,7 +62,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type ProductRow = InferTableType; + type ProductRow = TableRowType; const product: ProductRow = { id: 1, @@ -84,7 +84,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type PostRow = InferTableType; + type PostRow = TableRowType; const draftPost: PostRow = { id: 1, @@ -118,7 +118,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type EventRow = InferTableType; + type EventRow = TableRowType; const event1: EventRow = { id: 1, @@ -144,7 +144,7 @@ void describe('Type Inference Runtime Tests', () => { }, }); - type UserRow = InferTableType; + type UserRow = TableRowType; const mockQueryResults: UserRow[] = [ { id: 1, email: 'alice@example.com', nickname: 'alice', age: 30 }, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 4a85aa8f..7429b1c1 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -1,7 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; +import type { TableRowType } from '../components'; import { dumboSchema } from './index'; +import type { Expect, Equal } from '../../testing'; const { database, schema, table, column, index } = dumboSchema; const { Varchar } = SQL.column.type; @@ -144,8 +146,8 @@ void describe('dumboSchema', () => { const users = table('users', { columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), + id: column('id', Varchar('max'), { primaryKey: true, notNull: true }), + email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, }); @@ -162,8 +164,9 @@ const multiSchemaDb = database('myapp', { public: schema('public', { users: table('users', { columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), + id: column('id', Varchar('max'), { primaryKey: true, notNull: true }), + email: column('email', Varchar('max'), { notNull: true }), + name: column('name', Varchar('max')), }, }), }), @@ -180,5 +183,10 @@ const multiSchemaDb = database('myapp', { // Access using name-based maps const publicSchema = multiSchemaDb.schemas.public; -const usersTable = publicSchema?.tables.get('users'); -export const emailColumn = usersTable?.columns.get('email'); +const usersTable = publicSchema.tables.users; + +type Users = TableRowType; + +type _IdColumnIsNonNullableString = Expect>; +type _EmailColumnIsNonNullableString = Expect>; +type _NameColumnIsNullableString = Expect>; diff --git a/src/packages/dumbo/src/core/testing/index.ts b/src/packages/dumbo/src/core/testing/index.ts new file mode 100644 index 00000000..d2afd3d7 --- /dev/null +++ b/src/packages/dumbo/src/core/testing/index.ts @@ -0,0 +1 @@ +export * from './typesTesting'; diff --git a/src/packages/dumbo/src/core/testing/typesTesting.ts b/src/packages/dumbo/src/core/testing/typesTesting.ts new file mode 100644 index 00000000..b38f602e --- /dev/null +++ b/src/packages/dumbo/src/core/testing/typesTesting.ts @@ -0,0 +1,5 @@ +export type Expect = T; +export type Equal = + (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 + ? true + : false; From d56eef4140e0f4e096ab2833a7de75c78f8e226c Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 13:11:58 +0100 Subject: [PATCH 33/39] Fixed Pongo databases and collections identifiers to be unique urns --- src/packages/dumbo/src/core/schema/components/index.ts | 2 +- .../{typeInference.ts => tableTypesInference.ts} | 0 ...rence.type.spec.ts => tableTypesInference.type.spec.ts} | 4 ++-- ...rence.unit.spec.ts => tableTypesInference.unit.spec.ts} | 2 +- .../src/core/schema/dumboSchema/dumboSchema.unit.spec.ts | 6 +++--- .../src/core/collection/pongoCollectionSchemaComponent.ts | 7 +++++-- .../src/core/database/pongoDatabaseSchemaComponent.ts | 7 +++++-- 7 files changed, 17 insertions(+), 11 deletions(-) rename src/packages/dumbo/src/core/schema/components/{typeInference.ts => tableTypesInference.ts} (100%) rename src/packages/dumbo/src/core/schema/components/{typeInference.type.spec.ts => tableTypesInference.type.spec.ts} (98%) rename src/packages/dumbo/src/core/schema/components/{typeInference.unit.spec.ts => tableTypesInference.unit.spec.ts} (98%) diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts index fd674eb6..6aad254e 100644 --- a/src/packages/dumbo/src/core/schema/components/index.ts +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -9,7 +9,7 @@ export * from './databaseSchemaComponent'; export * from './databaseSchemaSchemaComponent'; export * from './indexSchemaComponent'; export * from './tableSchemaComponent'; -export * from './typeInference'; +export * from './tableTypesInference'; export const schemaComponentURN = { database: DatabaseURN, diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts similarity index 100% rename from src/packages/dumbo/src/core/schema/components/typeInference.ts rename to src/packages/dumbo/src/core/schema/components/tableTypesInference.ts diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts similarity index 98% rename from src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts rename to src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts index d95c15ea..100cb294 100644 --- a/src/packages/dumbo/src/core/schema/components/typeInference.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts @@ -9,14 +9,14 @@ import type { TimestamptzToken, VarcharToken, } from '../../sql/tokens/columnTokens'; -import type { Expect, Equal } from '../../testing'; +import type { Equal, Expect } from '../../testing'; import { dumboSchema } from '../dumboSchema'; import type { InferColumnType, InferTableRow, TableColumnType, TableRowType, -} from './typeInference'; +} from './tableTypesInference'; const { table, column } = dumboSchema; const { Serial, BigSerial, Integer, BigInteger, Varchar, Timestamp, JSONB } = diff --git a/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.unit.spec.ts similarity index 98% rename from src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts rename to src/packages/dumbo/src/core/schema/components/tableTypesInference.unit.spec.ts index fbf2dfd3..9099cf97 100644 --- a/src/packages/dumbo/src/core/schema/components/typeInference.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.unit.spec.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; import { dumboSchema } from '../dumboSchema'; -import type { TableRowType } from './typeInference'; +import type { TableRowType } from './tableTypesInference'; const { table, column } = dumboSchema; const { Serial, Varchar, Integer, Timestamp, JSONB } = SQL.column.type; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 7429b1c1..23b1bb3a 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -1,9 +1,9 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; +import type { Equal, Expect } from '../../testing'; import type { TableRowType } from '../components'; import { dumboSchema } from './index'; -import type { Expect, Equal } from '../../testing'; const { database, schema, table, column, index } = dumboSchema; const { Varchar } = SQL.column.type; @@ -183,9 +183,9 @@ const multiSchemaDb = database('myapp', { // Access using name-based maps const publicSchema = multiSchemaDb.schemas.public; -const usersTable = publicSchema.tables.users; +const _usersTable = publicSchema.tables.users; -type Users = TableRowType; +type Users = TableRowType; type _IdColumnIsNonNullableString = Expect>; type _EmailColumnIsNonNullableString = Expect>; diff --git a/src/packages/pongo/src/core/collection/pongoCollectionSchemaComponent.ts b/src/packages/pongo/src/core/collection/pongoCollectionSchemaComponent.ts index 86b3af25..d41af64b 100644 --- a/src/packages/pongo/src/core/collection/pongoCollectionSchemaComponent.ts +++ b/src/packages/pongo/src/core/collection/pongoCollectionSchemaComponent.ts @@ -6,8 +6,11 @@ import { } from '@event-driven-io/dumbo'; import type { PongoCollectionSchema, PongoCollectionSQLBuilder } from '..'; +export type PongoCollectionURNType = 'sc:pongo:collection'; +export type PongoCollectionURN = `${PongoCollectionURNType}:${string}`; + export type PongoCollectionSchemaComponent = - SchemaComponent<'pongo:schema-component:collection'> & { + SchemaComponent & { collectionName: string; definition: PongoCollectionSchema; sqlBuilder: PongoCollectionSQLBuilder; @@ -31,7 +34,7 @@ export const PongoCollectionSchemaComponent = < }: PongoCollectionSchemaComponentOptions): PongoCollectionSchemaComponent => ({ ...schemaComponent( - 'pongo:schema-component:collection', + `sc:pongo:collection:${definition.name}`, migrationsOrSchemaComponents, ), sqlBuilder, diff --git a/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts b/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts index a01f0f48..26cb0d9a 100644 --- a/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts +++ b/src/packages/pongo/src/core/database/pongoDatabaseSchemaComponent.ts @@ -14,6 +14,9 @@ import { } from '../schema'; import type { PongoDocument } from '../typing'; +export type PongoDatabaseURNType = 'sc:dumbo:database'; +export type PongoDatabaseURN = `${PongoDatabaseURNType}:${string}`; + export type PongoDatabaseSchemaComponent< // eslint-disable-next-line @typescript-eslint/no-unused-vars DriverType extends DatabaseDriverType = DatabaseDriverType, @@ -21,7 +24,7 @@ export type PongoDatabaseSchemaComponent< string, PongoCollectionSchema >, -> = SchemaComponent<'pongo:schema-component:database'> & { +> = SchemaComponent & { definition: PongoDbSchema; collections: ReadonlyArray; @@ -54,7 +57,7 @@ export const PongoDatabaseSchemaComponent = < Object.values(definition.collections).map(collectionFactory) ?? []; return { - ...schemaComponent('pongo:schema-component:database', { + ...schemaComponent(`sc:dumbo:database:${definition.name}`, { components: collections, }), definition, From 47a5898eab0890df288c2d54e172203aece2c939 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 15:18:08 +0100 Subject: [PATCH 34/39] Added helper for TableColumnNames --- .../dumbo/src/core/schema/components/tableTypesInference.ts | 5 +++++ .../src/core/schema/dumboSchema/dumboSchema.unit.spec.ts | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts index 9afc9710..20e482e1 100644 --- a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts @@ -26,6 +26,11 @@ export type TableColumnType = : InferColumnType | null : unknown; +export type TableColumnNames = Exclude< + keyof T['columns'], + keyof ReadonlyMap +>; + export type InferTableRow = { [K in keyof Columns]: TableColumnType; }; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 23b1bb3a..582da823 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; import type { Equal, Expect } from '../../testing'; -import type { TableRowType } from '../components'; +import type { TableColumnNames, TableRowType } from '../components'; import { dumboSchema } from './index'; const { database, schema, table, column, index } = dumboSchema; @@ -190,3 +190,7 @@ type Users = TableRowType; type _IdColumnIsNonNullableString = Expect>; type _EmailColumnIsNonNullableString = Expect>; type _NameColumnIsNullableString = Expect>; + +type UserColumns = TableColumnNames; + +const _userColumns: UserColumns[] = ['id', 'email', 'name']; From 9ace79b5bb03970bd2c8f477a5be9227f65d377f Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Wed, 19 Nov 2025 15:47:08 +0100 Subject: [PATCH 35/39] Added a way to define primary key for table --- .../core/schema/components/tableSchemaComponent.ts | 5 +++++ .../src/core/schema/dumboSchema/dumboSchema.ts | 5 ++++- .../schema/dumboSchema/dumboSchema.unit.spec.ts | 13 +++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 0848dd25..46f659e1 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -12,6 +12,7 @@ import { IndexURNType, type IndexSchemaComponent, } from './indexSchemaComponent'; +import type { TableColumnNames } from './tableTypesInference'; export type TableURNType = 'sc:dumbo:table'; export type TableURN = `${TableURNType}:${string}`; @@ -28,6 +29,7 @@ export type TableSchemaComponent = Readonly<{ tableName: string; columns: ReadonlyMap & Columns; + primaryKey: TableColumnNames>[]; indexes: ReadonlyMap; addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; @@ -42,10 +44,12 @@ export const tableSchemaComponent = < >({ tableName, columns, + primaryKey, ...migrationsOrComponents }: { tableName: string; columns?: Columns; + primaryKey?: TableColumnNames>[]; } & SchemaComponentOptions): TableSchemaComponent => { columns ??= {} as Columns; @@ -60,6 +64,7 @@ export const tableSchemaComponent = < return { ...base, tableName, + primaryKey: primaryKey ?? [], get columns() { const columnsMap = mapSchemaComponentsOfType( base.components, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 61c203de..c41d1317 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -11,6 +11,7 @@ import { type DatabaseSchemaTables, indexSchemaComponent, type IndexSchemaComponent, + type TableColumnNames, type TableColumns, tableSchemaComponent, type TableSchemaComponent, @@ -60,16 +61,18 @@ const dumboTable = ( name: string, definition: { columns?: Columns; + primaryKey?: TableColumnNames>[]; indexes?: Record; } & SchemaComponentOptions, ): TableSchemaComponent => { - const { columns, indexes, ...options } = definition; + const { columns, indexes, primaryKey, ...options } = definition; const components = [...(indexes ? Object.values(indexes) : [])]; return tableSchemaComponent({ tableName: name, columns: columns ?? ({} as Columns), + primaryKey: primaryKey ?? [], components, ...options, }); diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 582da823..ac2293da 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -6,7 +6,7 @@ import type { TableColumnNames, TableRowType } from '../components'; import { dumboSchema } from './index'; const { database, schema, table, column, index } = dumboSchema; -const { Varchar } = SQL.column.type; +const { Varchar, JSONB } = SQL.column.type; void describe('dumboSchema', () => { void it('should create a column', () => { @@ -164,16 +164,18 @@ const multiSchemaDb = database('myapp', { public: schema('public', { users: table('users', { columns: { - id: column('id', Varchar('max'), { primaryKey: true, notNull: true }), + id: column('id', Varchar('max'), { notNull: true }), email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), + metadata: column('metadata', JSONB<{ preferences: string[] }>()), }, + primaryKey: ['id'], }), }), analytics: schema('analytics', { events: table('events', { columns: { - id: column('id', Varchar('max')), + id: column('id', Varchar('max'), { notNull: true, primaryKey: true }), userId: column('user_id', Varchar('max')), timestamp: column('timestamp', Varchar('max')), }, @@ -190,7 +192,10 @@ type Users = TableRowType; type _IdColumnIsNonNullableString = Expect>; type _EmailColumnIsNonNullableString = Expect>; type _NameColumnIsNullableString = Expect>; +type _MetadataColumnIsNullableObject = Expect< + Equal +>; type UserColumns = TableColumnNames; -const _userColumns: UserColumns[] = ['id', 'email', 'name']; +const _userColumns: UserColumns[] = ['id', 'email', 'name', 'metadata']; From 727280ef423c1b1ddf6b5038e5661a59db1dc381 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Thu, 20 Nov 2025 14:49:04 +0100 Subject: [PATCH 36/39] Added foreign key support --- plan.md | 1899 +++++++++++++---- .../components/databaseSchemaComponent.ts | 4 + .../components/foreignKeys/foreignKeyTypes.ts | 68 + .../foreignKeys/foreignKeyValidation.ts | 186 ++ .../foreignKeyValidation.type.spec.ts | 572 +++++ .../schema/components/foreignKeys/index.ts | 3 + .../dumbo/src/core/schema/components/index.ts | 1 + .../schema/components/tableSchemaComponent.ts | 57 +- .../schema/components/tableTypesInference.ts | 14 + .../core/schema/dumboSchema/dumboSchema.ts | 38 +- .../dumboSchema/dumboSchema.unit.spec.ts | 158 ++ .../dumbo/src/core/testing/typesTesting.ts | 1 + todo.md | 71 + 13 files changed, 2672 insertions(+), 400 deletions(-) create mode 100644 src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts create mode 100644 src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts create mode 100644 src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts create mode 100644 todo.md diff --git a/plan.md b/plan.md index 9382dc72..f3f746d8 100644 --- a/plan.md +++ b/plan.md @@ -1,461 +1,1610 @@ -# Dumbo Database Schema Builder Implementation Plan (Simplified) +# Foreign Key Type Validation Implementation Plan ## Overview -Create a simple builder API for defining database schemas in dumbo, following the PongoSchema pattern but using const functions and reusing existing schema components. +Implement compile-time type validation for foreign keys in Dumbo schema definitions. The validation happens at the database/schema level (top-down), ensuring all foreign key references are valid without requiring `as const` annotations. -## Key Principles +## Goals -- **No overengineering** - Keep it simple -- **Use const instead of function** - Follow the existing pattern -- **Object parameters** - No function overloading, use objects with optional fields -- **No useless comments**. +- ✅ Type-safe foreign key definitions with full intellisense +- ✅ Validate `columns` array contains only columns from the defining table +- ✅ Validate `references` array contains only valid `schema.table.column` paths +- ✅ Support composite foreign keys (multiple columns mapping to multiple references) +- ✅ Support self-referential foreign keys +- ✅ Clear, actionable error messages when validation fails +- ✅ No runtime validation - pure TypeScript type-level implementation +- ✅ No `as const` required - use generic inference -## Implementation Steps +## Technical Approach -### Step 1: Create Constants File +### Type System Strategy -**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/constants.ts` +1. Use generic inference in `table()` function to capture exact tuple types from array literals +2. Store foreign key definitions in table schema component +3. Validate all foreign keys at `database()` level using recursive type utilities +4. Show all validation errors (optimize to first error only if performance issues arise) + +### Validation Flow + +``` +database() called + → Extract all schemas + → Extract all tables from each schema + → Extract all columns from each table + → Build union of all valid column references: 'schema.table.column' + → For each table with foreignKeys: + → Validate each FK's columns array against table's columns + → Validate each FK's references array against all valid column references + → Validate columns.length === references.length + → Collect all errors with helpful messages + → If any errors: database() RETURN TYPE becomes ValidationResult + → If no errors: database() RETURN TYPE is DatabaseSchemaComponent + → TypeScript shows errors at database() call site when types don't match + → Use Expect/ExpectError type tests to verify validation works +``` + +**Why Return Types:** +TypeScript cannot validate function parameters before type inference completes. The solution is conditional return types - when validation fails, the return type becomes an error object (`{ valid: false; error: "..." }`) which is incompatible with `DatabaseSchemaComponent`, causing type errors at the assignment site. + +## Quality Gates + +**CRITICAL: After EVERY step, run the following as subagents (to reduce context):** + +1. `cd src & npm run fix` - Fix linting issues +2. `cd src & npm run build:ts` - Ensure TypeScript compiles +3. `cd src & npm run test:unit` - Run all tests + +YOU MUST run them through sub agents, not bash. + +**All three must pass before proceeding to the next step.** + +If any fail and cannot be fixed automatically, **STOP and ask for help**. + +## Validation Protocol + +For EACH step: + +1. Make changes to the code +2. Run quality gates ONCE via subagents in parallel: + - `npm run fix` + - `npm run build:ts` + - `npm run test:unit` +3. If any gate fails, fix the issue and trust that the fix worked (don't rerun) +4. Commit immediately after fixes +5. Mark step as complete + +NEVER re-run the same command twice. Trust subagent results. + +## Code Standards + +- Use `describe`/`it` for runtime tests (matching existing test patterns) +- For type tests (\*.type.spec.ts), use simple type assertions at top level (no `void it()` wrappers - see tableTypesInference.type.spec.ts) +- Never import js files, always use typescript, +- No classes unless absolutely necessary +- Avoid over-complexity +- No comments unless they add genuine value +- Follow existing codebase patterns and conventions +- Prefer simple, clean, maintainable solutions + +## Implementation Phases + +### Phase 1: Foundation - Type Extraction Utilities + +Build the base type utilities to extract schema structure information. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (new) +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (new) +- `src/packages/dumbo/src/core/testing/typesTesting.ts` (modify - add ExpectError) + +**Deliverables:** + +- `ExpectError` helper type +- `ExtractSchemaNames` - extract schema names from database +- `ExtractTableNames` - extract table names from schema +- Type tests for all extraction utilities (reuse existing `TableColumnNames` for columns) + +### Phase 2: Column Reference Generation + +Build utilities to generate all valid column reference paths. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (modify) +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) + +**Deliverables:** + +- `AllColumnReferences` - generate union of all valid 'schema.table.column' strings +- Type tests for single-schema and multi-schema databases + +### Phase 3: Foreign Key Type Definition + +Define the foreign key structure and add it to table schema component. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (modify) +- `src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts` (modify) +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) + +**Deliverables:** + +- `ForeignKeyDefinition` type +- Updated `TableSchemaComponent` with generic FK parameter +- Type tests for FK structure + +### Phase 4: Single Foreign Key Validation + +Implement validation logic for a single foreign key definition. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts` (new) +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) + +**Deliverables:** + +- `ValidateForeignKeyColumns` - validate columns array +- `ValidateForeignKeyReferences` - validate references array +- `ValidateForeignKeyLength` - validate columns.length === references.length +- `ValidateSingleForeignKey` - combine all validations +- Comprehensive error message types with helpful suggestions +- Type tests for valid and invalid scenarios + +### Phase 5: Full Database Validation + +Implement database-level validation that checks all foreign keys across all tables. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts` (modify) +- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) + +**Deliverables:** + +- `ValidateTableForeignKeys` - validate all FKs in a table +- `ValidateSchemaForeignKeys` - validate all FKs in a schema +- `ValidateDatabaseForeignKeys` - validate all FKs in entire database +- Type tests for multi-table, multi-schema validation + +### Phase 6: Integration with dumboSchema + +Wire up validation to the `database()` and `schema()` functions. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts` (modify) +- `src/packages/dumbo/src/core/schema/components/index.ts` (modify - re-export FK types) +- `src/packages/dumbo/src/core/schema/foreignKeys/index.ts` (new - barrel export) + +**Deliverables:** + +- Updated `dumboTable` signature to accept and capture `foreignKeys` generic +- Updated `dumboDatabase` signature with FK validation constraint +- Updated `dumboDatabaseSchema` signature with FK validation constraint +- Proper type exports + +### Phase 7: Real-World Testing + +Test with actual schema definitions and ensure everything works. + +**Files to create/modify:** + +- `src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts` (modify) + +**Deliverables:** + +- Working examples with valid foreign keys +- Multiple test cases (single FK, composite FK, self-referential, multiple FKs, cross-schema) +- Update existing multiSchemaDb example to verify it works correctly + +--- + +## Detailed Step-by-Step Prompts + +Each prompt below is self-contained and builds on the previous work. They follow TDD principles: write type tests first, then implement the types to pass the tests. + +--- + +### Step 1: Add ExpectError Helper + +**Context:** We need a type helper to test that validation errors are properly generated. This helper will be used throughout our type tests. + +**Prompt:** + +```` +Add an `ExpectError` helper type to src/packages/dumbo/src/core/testing/typesTesting.ts. + +The helper should: +- Accept a type parameter T +- Return true if T extends { valid: false } +- Be used like: type Test = ExpectError + +Add it below the existing Expect and Equal helpers, and export it. + +Example usage: +```typescript +type InvalidCase = { valid: false, error: string }; +type Test = ExpectError; +```` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- ExpectError type added and exported +- No breaking changes to existing tests +- All quality gates pass + +--- + +### Step 2: Create Foreign Key Types File Structure + +**Context:** Set up the base file structure for foreign key validation types and tests. + +**Prompt:** +``` + +Create two new files: + +1. src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts + + - Add placeholder comment: `// Foreign key type definitions` + +2. src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts + - Import Expect, Equal, ExpectError from '../../testing' + - Add comment placeholders for test sections: + - Schema Structure Extraction + - Column Reference Generation + - Foreign Key Definition Structure + - Foreign Key Validation - Valid Cases + - Foreign Key Validation - Invalid Cases + - Integration Tests + - Follow the pattern from tableTypesInference.type.spec.ts (simple type assertions, no describe/it blocks) + +Ensure the test file is recognized by the test runner. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Both files created with proper structure +- Test file runs without errors +- File structure follows existing patterns +- All quality gates pass + +--- + +### Step 3: Implement ExtractSchemaNames Type Utility + +**Context:** We need to extract schema names from a database type to build validation logic. + +**Prompt:** +``` + +Following TDD approach, add type tests then implement ExtractSchemaNames utility. + +In foreignKeyValidation.type.spec.ts, add under 'Schema Structure Extraction' comment section: + +```typescript +// Schema Structure Extraction + +// ExtractSchemaNames - single schema +type _DB1 = DatabaseSchemaComponent<{ + public: DatabaseSchemaSchemaComponent; +}>; +type _Test1 = Expect, "public">>; + +// ExtractSchemaNames - multi schema +type _DB2 = DatabaseSchemaComponent<{ + public: DatabaseSchemaSchemaComponent; + analytics: DatabaseSchemaSchemaComponent; +}>; +type _Test2 = Expect, "public" | "analytics">>; +``` + +Then in foreignKeyTypes.ts, implement: + +```typescript +import type { DatabaseSchemaComponent, DatabaseSchemas } from "../components"; + +export type ExtractSchemaNames = + DB extends DatabaseSchemaComponent + ? keyof Schemas & string + : never; +``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ExtractSchemaNames correctly extracts schema names +- All quality gates pass + +--- + +### Step 4: Implement ExtractTableNames Type Utility + +**Context:** Extract table names from a schema to build validation logic. + +**Prompt:** +``` + +Following TDD approach, add type tests then implement ExtractTableNames utility. + +In foreignKeyValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: + +```typescript +void it("should extract table names from single-table schema", () => { + type Schema = DatabaseSchemaSchemaComponent<{ + users: TableSchemaComponent; + }>; + + type Result = ExtractTableNames; + type Test = Expect>; +}); + +void it("should extract table names from multi-table schema", () => { + type Schema = DatabaseSchemaSchemaComponent<{ + users: TableSchemaComponent; + posts: TableSchemaComponent; + comments: TableSchemaComponent; + }>; + + type Result = ExtractTableNames; + type Test = Expect>; +}); +``` + +Then in foreignKeyTypes.ts, implement: + +```typescript +import type { + DatabaseSchemaSchemaComponent, + DatabaseSchemaTables, +} from "../components"; + +export type ExtractTableNames = + Schema extends DatabaseSchemaSchemaComponent< + infer Tables extends DatabaseSchemaTables + > + ? keyof Tables & string + : never; +``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ExtractTableNames correctly extracts table names +- All quality gates pass + +--- + +### Step 5: Implement ExtractColumnNames Type Utility + +**Context:** Extract column names from a table to validate foreign key columns array. + +**Prompt:** +``` + +Following TDD approach, add type tests then implement ExtractColumnNames utility. + +In foreignKeyValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: + +```typescript +void it("should extract column names from single-column table", () => { + type Table = TableSchemaComponent<{ + id: AnyColumnSchemaComponent; + }>; + + type Result = ExtractColumnNames; + type Test = Expect>; +}); + +void it("should extract column names from multi-column table", () => { + type Table = TableSchemaComponent<{ + id: AnyColumnSchemaComponent; + email: AnyColumnSchemaComponent; + name: AnyColumnSchemaComponent; + created_at: AnyColumnSchemaComponent; + }>; + + type Result = ExtractColumnNames
; + type Test = Expect>; +}); +``` + +Then in foreignKeyTypes.ts, implement: + +```typescript +import type { + TableSchemaComponent, + TableColumns, + TableColumnNames, +} from "../components"; + +export type ExtractColumnNames
= + Table extends TableSchemaComponent + ? TableColumnNames
+ : never; +``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ExtractColumnNames reuses existing TableColumnNames utility +- All quality gates pass + +--- + +### Step 6: Implement AllColumnReferences Type Utility (Single Schema) + +**Context:** Generate all valid column reference paths in 'schema.table.column' format. Start with single schema case. + +**Prompt:** +``` + +Following TDD approach, add type test then implement AllColumnReferences for single schema. + +In foreignKeyValidation.type.spec.ts, add to 'Column Reference Generation' describe block: + +```typescript +import { SQL } from "../../../sql"; +import { dumboSchema } from "../../dumboSchema"; + +const { database, schema, table, column } = dumboSchema; +const { Varchar } = SQL.column.type; + +void it("should generate column references for single schema with one table", () => { + const db = database("test", { + public: schema("public", { + users: table("users", { + columns: { + id: column("id", Varchar("max")), + email: column("email", Varchar("max")), + }, + }), + }), + }); + + type Result = AllColumnReferences; + type Test = Expect>; +}); + +void it("should generate column references for single schema with multiple tables", () => { + const db = database("test", { + public: schema("public", { + users: table("users", { + columns: { + id: column("id", Varchar("max")), + email: column("email", Varchar("max")), + }, + }), + posts: table("posts", { + columns: { + id: column("id", Varchar("max")), + title: column("title", Varchar("max")), + user_id: column("user_id", Varchar("max")), + }, + }), + }), + }); + + type Result = AllColumnReferences; + type Test = Expect< + Equal< + Result, + | "public.users.id" + | "public.users.email" + | "public.posts.id" + | "public.posts.title" + | "public.posts.user_id" + > + >; +}); +``` + +Then in foreignKeyTypes.ts, implement: + +```typescript +export type AllColumnReferences = + DB extends DatabaseSchemaComponent + ? { + [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< + infer Tables + > + ? { + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns + > + ? { + [ColumnName in keyof Columns]: `${SchemaName & + string}.${TableName & string}.${ColumnName & string}`; + }[keyof Columns] + : never; + }[keyof Tables] + : never; + }[keyof Schemas] + : never; +``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- AllColumnReferences generates correct paths for single schema +- All quality gates pass + +--- + +### Step 7: Test AllColumnReferences with Multi-Schema Database + +**Context:** Verify AllColumnReferences works with multiple schemas. + +**Prompt:** +``` + +Add type test for multi-schema database to verify AllColumnReferences works correctly. + +In foreignKeyValidation.type.spec.ts, add to 'Column Reference Generation' describe block: + +```typescript +void it("should generate column references for multi-schema database", () => { + const db = database("test", { + public: schema("public", { + users: table("users", { + columns: { + id: column("id", Varchar("max")), + email: column("email", Varchar("max")), + }, + }), + }), + analytics: schema("analytics", { + events: table("events", { + columns: { + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), + event_type: column("event_type", Varchar("max")), + }, + }), + }), + }); + + type Result = AllColumnReferences; + type Test = Expect< + Equal< + Result, + | "public.users.id" + | "public.users.email" + | "analytics.events.id" + | "analytics.events.user_id" + | "analytics.events.event_type" + > + >; +}); +``` + +Run tests to ensure they pass. No implementation changes needed - this verifies existing implementation works. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type test added and passes +- AllColumnReferences works with multiple schemas +- All quality gates pass + +--- + +### Step 8: Define ForeignKeyDefinition Type + +**Context:** Define the structure for foreign key definitions that tables will use. + +**Prompt:** +``` + +Add the ForeignKeyDefinition type to foreignKeyTypes.ts and create a basic type test. + +In foreignKeyTypes.ts, add: + +```typescript +export type ForeignKeyDefinition = { + readonly columns: readonly string[]; + readonly references: readonly string[]; +}; +``` + +In foreignKeyValidation.type.spec.ts, add to 'Foreign Key Definition Structure' describe block: + +```typescript +void it("should accept valid foreign key definition", () => { + type FK = ForeignKeyDefinition; + + const validFK: FK = { + columns: ["user_id"], + references: ["public.users.id"], + }; + + type ColumnsType = typeof validFK.columns; + type Test = Expect>; +}); + +void it("should accept composite foreign key definition", () => { + type FK = ForeignKeyDefinition; + + const compositeFK: FK = { + columns: ["user_id", "tenant_id"], + references: ["public.users.id", "public.users.tenant_id"], + }; + + type ColumnsType = typeof compositeFK.columns; + type ReferencesType = typeof compositeFK.references; + type Test1 = Expect>; + type Test2 = Expect>; +}); +``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- ForeignKeyDefinition type added +- Basic structure tests pass +- All quality gates pass + +--- + +### Step 9: Update TableSchemaComponent to Include Foreign Keys + +**Context:** Add foreign keys support to the table schema component type definition. + +**Prompt:** +``` + +Update TableSchemaComponent to support an optional foreignKeys property. + +In src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts: + +1. Import ForeignKeyDefinition: + +```typescript +import type { ForeignKeyDefinition } from "../foreignKeys/foreignKeyTypes"; +``` + +2. Add generic parameter for foreign keys and foreignKeys property: + +```typescript +export type TableSchemaComponent< + Columns extends TableColumns = TableColumns, + ForeignKeys extends readonly ForeignKeyDefinition[] = readonly ForeignKeyDefinition[] +> = SchemaComponent< + TableURN, + Readonly<{ + tableName: string; + columns: ReadonlyMap & Columns; + primaryKey: TableColumnNames>[]; + foreignKeys?: ForeignKeys; + indexes: ReadonlyMap; + addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; + addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; + }> +>; +``` + +3. Update AnyTableSchemaComponent: + +```typescript +export type AnyTableSchemaComponent = TableSchemaComponent; +``` + +4. Update tableSchemaComponent function signature and implementation to accept and return foreignKeys. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- TableSchemaComponent updated with foreignKeys support +- Generic parameter added for type safety +- No breaking changes to existing code +- All quality gates pass + +--- + +### Step 10: Update dumboTable to Accept Foreign Keys + +**Context:** Update the dumboTable function to accept and capture foreign keys with generic inference. + +**Prompt:** +``` + +Update dumboTable function in dumboSchema.ts to accept foreignKeys parameter. + +In src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts: + +1. Import ForeignKeyDefinition at the top: ```typescript -// Special key for default schema -export const DEFAULT_SCHEMA = Symbol.for("dumbo.defaultSchema"); +import type { ForeignKeyDefinition } from "../foreignKeys/foreignKeyTypes"; +``` + +2. Update dumboTable function to accept foreignKeys in definition parameter and pass it to tableSchemaComponent. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. -// Database-specific defaults (for reference/future use) -export const DATABASE_DEFAULTS = { - PostgreSQL: { defaultDatabase: "postgres", defaultSchema: "public" }, - MySQL: { defaultDatabase: null, defaultSchema: null }, - SQLite: { defaultDatabase: null, defaultSchema: "main" }, - SqlServer: { defaultDatabase: "master", defaultSchema: "dbo" }, -} as const; ``` -**Validation**: +**Acceptance Criteria:** +- dumboTable accepts foreignKeys parameter +- Generic inference captures exact FK types +- No breaking changes +- All quality gates pass -- [ ] Compiles -- [ ] No linter errors +--- -### Step 2: Create Builder Functions +### Step 11: Implement ValidateForeignKeyLength -**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/index.ts` +**Context:** Validate that foreign key columns and references arrays have matching lengths. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateForeignKeyLength. + +Create new file src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts with validation result types and ValidateForeignKeyLength. + +In foreignKeyValidation.type.spec.ts, add to 'Foreign Key Validation - Invalid Cases' describe block: ```typescript -import { - columnSchemaComponent, - type ColumnSchemaComponent, - indexSchemaComponent, - type IndexSchemaComponent, - tableSchemaComponent, - type TableSchemaComponent, - databaseSchemaSchemaComponent, - type DatabaseSchemaSchemaComponent, - databaseSchemaComponent, - type DatabaseSchemaComponent, -} from "../components"; -import type { SchemaComponentOptions } from "../schemaComponent"; -import { DEFAULT_SCHEMA, DATABASE_DEFAULTS } from "./constants"; - -// Column builder - simple const -const dumboColumn = ( - name: string, - options?: SchemaComponentOptions -): ColumnSchemaComponent => - columnSchemaComponent({ - columnName: name, - ...options, - }); +void it("should error when columns and references have different lengths", () => { + type FK = { + columns: ["user_id", "tenant_id"]; + references: ["public.users.id"]; + }; + + type Result = ValidateForeignKeyLength; + type Test = ExpectError; +}); -// Index builder - simple const -const dumboIndex = ( - name: string, - columnNames: string[], - options?: { unique?: boolean } & SchemaComponentOptions -): IndexSchemaComponent => - indexSchemaComponent({ - indexName: name, - columnNames, - isUnique: options?.unique ?? false, - ...options, - }); +void it("should pass when columns and references have same length", () => { + type FK1 = { + columns: ["user_id"]; + references: ["public.users.id"]; + }; + + type FK2 = { + columns: ["user_id", "tenant_id"]; + references: ["public.users.id", "public.users.tenant_id"]; + }; + + type Result1 = ValidateForeignKeyLength; + type Result2 = ValidateForeignKeyLength; + type Test1 = Expect>; + type Test2 = Expect>; +}); +``` -// Table builder - takes columns and indexes as objects -const dumboTable = ( - name: string, - definition: { - columns?: Record; - indexes?: Record; - } & SchemaComponentOptions -): TableSchemaComponent => { - const { columns, indexes, ...options } = definition; - - const components = [ - ...(columns ? Object.values(columns) : []), - ...(indexes ? Object.values(indexes) : []), - ]; - - return tableSchemaComponent({ - tableName: name, - components, - ...options, - }); -}; +After implementation, run quality gates as subagents: -// Schema builder - name is optional for default schema -const dumboDatabaseSchema = ( - nameOrTables: string | Record, - tables?: Record, - options?: SchemaComponentOptions -): DatabaseSchemaSchemaComponent => { - if (typeof nameOrTables === "string") { - // Named schema: dumboDatabaseSchema('public', {...}) - const tableComponents = Object.values(tables || {}); - return databaseSchemaSchemaComponent({ - schemaName: nameOrTables, - components: tableComponents, - ...options, - }); - } else { - // Default schema: dumboDatabaseSchema({...}) - const tableComponents = Object.values(nameOrTables || {}); - return databaseSchemaSchemaComponent({ - schemaName: "", // Will be replaced with default - components: tableComponents, - }); - } -}; +1. npm run fix +2. npm run build:ts +3. npm run test:unit -// Helper for creating from array of names -dumboDatabaseSchema.from = ( - schemaName: string | undefined, - tableNames: string[] -): DatabaseSchemaSchemaComponent => { - const tables = tableNames.reduce((acc, tableName) => { - acc[tableName] = dumboTable(tableName, {}); - return acc; - }, {} as Record); - - return schemaName - ? dumboDatabaseSchema(schemaName, tables) - : dumboDatabaseSchema(tables); -}; +All must pass before proceeding. -// Database builder - name is optional for default database -const dumboDatabase = ( - nameOrSchemas: - | string - | Record, - schemas?: Record, - options?: { defaultSchemaName?: string } & SchemaComponentOptions -): DatabaseSchemaComponent => { - let databaseName: string; - let schemaMap: Record; - let dbOptions: typeof options; - - if (typeof nameOrSchemas === "string") { - // Named database: dumboDatabase('myapp', {...}) - databaseName = nameOrSchemas; - schemaMap = schemas || {}; - dbOptions = options; - } else { - // Default database: dumboDatabase({...}) - databaseName = "database"; // Default name - schemaMap = nameOrSchemas; - dbOptions = schemas as typeof options; - } - - // Process schemas, handling DEFAULT_SCHEMA - const schemaComponents: DatabaseSchemaSchemaComponent[] = []; - - for (const [key, schemaComponent] of Object.entries(schemaMap)) { - if (key === DEFAULT_SCHEMA.toString() || key === String(DEFAULT_SCHEMA)) { - // This is the default schema - replace its name - const defaultSchemaName = dbOptions?.defaultSchemaName || "public"; - schemaComponents.push( - databaseSchemaSchemaComponent({ - schemaName: defaultSchemaName, - components: Array.from(schemaComponent.components.values()), - migrations: schemaComponent.migrations, - }) - ); - } else { - schemaComponents.push(schemaComponent); - } - } - - return databaseSchemaComponent({ - databaseName, - components: schemaComponents, - ...dbOptions, - }); -}; +``` -// Helper for creating from array of names -dumboDatabase.from = ( - databaseName: string | undefined, - schemaNames: string[] -): DatabaseSchemaComponent => { - const schemas = schemaNames.reduce((acc, schemaName) => { - acc[schemaName] = dumboDatabaseSchema(schemaName, {}); - return acc; - }, {} as Record); - - return databaseName - ? dumboDatabase(databaseName, schemas) - : dumboDatabase(schemas); -}; +**Acceptance Criteria:** +- Type tests added and pass +- ValidateForeignKeyLength correctly validates array lengths +- All quality gates pass -// Main export - similar to pongoSchema -export const dumboSchema = { - database: dumboDatabase, - schema: dumboDatabaseSchema, - table: dumboTable, - column: dumboColumn, - index: dumboIndex, - DEFAULT_SCHEMA, - DATABASE_DEFAULTS, -}; +--- + +### Step 12: Implement ValidateForeignKeyColumns + +**Context:** Validate that all columns in a foreign key exist in the table. + +**Prompt:** ``` -**Reference files**: +Following TDD, add type tests then implement ValidateForeignKeyColumns. + +In foreignKeyValidation.type.spec.ts, add tests to both valid and invalid cases describe blocks. -- `/src/packages/pongo/src/core/schema/index.ts` (pattern to follow) -- `/src/packages/dumbo/src/core/schema/components/` (components to reuse) +In foreignKeyValidation.ts, implement ValidateForeignKeyColumns with helper types to check if all elements of a tuple are in a union and to find invalid columns. -**Validation**: +After implementation, run quality gates as subagents: -- [ ] Compiles -- [ ] No linter errors -- [ ] Exports work correctly +1. npm run fix +2. npm run build:ts +3. npm run test:unit -### Step 3: Update Main Schema Export +All must pass before proceeding. -**File**: `/src/packages/dumbo/src/core/schema/index.ts` +``` -Add to existing exports: +**Acceptance Criteria:** +- Type tests added and pass +- ValidateForeignKeyColumns correctly validates column existence +- Helpful error messages show which columns are invalid +- All quality gates pass + +--- + +### Step 13: Implement ValidateForeignKeyReferences + +**Context:** Validate that all references in a foreign key point to valid schema.table.column paths. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateForeignKeyReferences. + +In foreignKeyValidation.type.spec.ts, add tests for valid and invalid reference scenarios. + +In foreignKeyValidation.ts, implement ValidateForeignKeyReferences with helper to find invalid references. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ValidateForeignKeyReferences correctly validates reference paths +- Helpful error messages show invalid references and available options +- All quality gates pass + +--- + +### Step 14: Implement ValidateSingleForeignKey (Combine Validations) + +**Context:** Combine all FK validations into a single validation function. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateSingleForeignKey that combines all validations. + +In foreignKeyValidation.type.spec.ts, add tests for complete FK validation covering valid cases and all error scenarios. + +In foreignKeyValidation.ts, implement ValidateSingleForeignKey that chains length, columns, and references validation. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ValidateSingleForeignKey checks all validation rules in order +- Returns first error encountered (fail fast) +- All quality gates pass + +--- + +### Step 15: Implement ValidateTableForeignKeys + +**Context:** Validate all foreign keys defined in a single table. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateTableForeignKeys. + +In foreignKeyValidation.type.spec.ts, add tests for tables with no FKs, single FK, and multiple FKs. + +In foreignKeyValidation.ts, implement ValidateTableForeignKeys with helper to iterate through FK array. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ValidateTableForeignKeys handles tables with 0, 1, or multiple FKs +- Returns first error encountered across all FKs +- All quality gates pass + +--- + +### Step 16: Implement ValidateSchemaForeignKeys + +**Context:** Validate all foreign keys across all tables in a schema. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateSchemaForeignKeys. + +In foreignKeyValidation.type.spec.ts, add tests for schemas with multiple tables and FKs. + +In foreignKeyValidation.ts, implement ValidateSchemaForeignKeys with helper to iterate through tables. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ValidateSchemaForeignKeys iterates through all tables +- Returns first error encountered +- All quality gates pass + +--- + +### Step 17: Implement ValidateDatabaseForeignKeys + +**Context:** Validate all foreign keys across all schemas in the database - the top-level validation. + +**Prompt:** +``` + +Following TDD, add type tests then implement ValidateDatabaseForeignKeys. + +In foreignKeyValidation.type.spec.ts, add to 'Integration Tests' describe block tests for complete database validation including self-referential FKs. + +In foreignKeyValidation.ts, implement ValidateDatabaseForeignKeys that validates all schemas using AllColumnReferences. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Type tests added and pass +- ValidateDatabaseForeignKeys validates entire database +- Handles self-referential FKs correctly +- Returns first error encountered +- All quality gates pass + +--- + +### Step 18: Create Foreign Keys Barrel Export + +**Context:** Create an index file to export all foreign key types cleanly. + +**Prompt:** +``` + +Create src/packages/dumbo/src/core/schema/foreignKeys/index.ts to export all foreign key types. + +Update src/packages/dumbo/src/core/schema/components/index.ts to re-export FK types. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. -```typescript -export * from "./dumboSchema"; ``` -**Validation**: +**Acceptance Criteria:** +- Barrel export file created +- All FK types exported from components/index.ts +- No circular dependency issues +- All quality gates pass + +--- + +### Step 19: Wire Validation to database() Function -- [ ] Export is accessible -- [ ] No circular dependencies +**Context:** Add conditional return types to database() so invalid FKs cause type errors at the call site. -### Step 4: Create Unit Tests +**Implementation Strategy:** -**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts` +Use conditional return types on `database()` function overloads: ```typescript -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { dumboSchema } from "./index"; - -describe("dumboSchema", () => { - it("should create a column", () => { - const col = dumboSchema.column("id"); - assert.strictEqual(col.columnName, "id"); - }); +function dumboDatabase(schemas: Schemas): + ValidateDatabaseForeignKeys> extends { valid: true } + ? DatabaseSchemaComponent + : ValidateDatabaseForeignKeys>; +``` - it("should create an index", () => { - const idx = dumboSchema.index("idx_email", ["email"]); - assert.strictEqual(idx.indexName, "idx_email"); - assert.strictEqual(idx.isUnique, false); - }); +**How it works:** +1. User calls `const db = database({...})` +2. TypeScript infers exact `Schemas` type from literal +3. Return type is evaluated: + - If validation passes: returns `DatabaseSchemaComponent` + - If validation fails: returns `{ valid: false; error: "..." }` +4. Error object is incompatible with `DatabaseSchemaComponent` +5. Type error appears at `const db = database(...)` line +6. Error message shows which validation failed - it("should create a unique index", () => { - const idx = dumboSchema.index("idx_email", ["email"], { unique: true }); - assert.strictEqual(idx.indexName, "idx_email"); - assert.strictEqual(idx.isUnique, true); - }); +**No `as const` needed** - Generic inference on `table()` already captures exact tuple types from array literals. - it("should create a table with columns and indexes", () => { - const tbl = dumboSchema.table("users", { +**Prompt:** +``` + +IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. + +## Part 1: Add Failing Type Tests + +Add to foreignKeyValidation.type.spec.ts: + +```typescript +// TEST: Invalid column should cause type error at database() call +const _dbInvalidColumn = database('test', { + public: schema('public', { + posts: table('posts', { columns: { - id: dumboSchema.column("id"), - email: dumboSchema.column("email"), + id: column('id', Varchar('max')), + user_id: column('id', Varchar('max')), }, - indexes: { - idx_email: dumboSchema.index("idx_email", ["email"]), + foreignKeys: [ + { columns: ['invalid_col'], references: ['public.users.id'] }, + ], + }), + }), +}); + +type _InvalidColResult = typeof _dbInvalidColumn; +type _Test_InvalidColumn = Expect< + Equal<_InvalidColResult, DatabaseSchemaComponent> +>; // This should FAIL because _InvalidColResult is error type + +// TEST: Valid FK should work +const _dbValid = database('test', { + public: schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), }, - }); - - assert.strictEqual(tbl.tableName, "users"); - assert.strictEqual(tbl.columns.size, 2); - assert.strictEqual(tbl.indexes.size, 1); - assert.ok(tbl.columns.has("id")); - assert.ok(tbl.columns.has("email")); - assert.ok(tbl.indexes.has("idx_email")); - }); + foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + }), + }), +}); - it("should create a named schema", () => { - const sch = dumboSchema.schema("public", { - users: dumboSchema.table("users", { - columns: { - id: dumboSchema.column("id"), - }, - }), - }); +type _ValidResult = typeof _dbValid; +type _Test_Valid = Expect< + Equal<_ValidResult, DatabaseSchemaComponent> +>; // This should PASS +``` - assert.strictEqual(sch.schemaName, "public"); - assert.strictEqual(sch.tables.size, 1); - assert.ok(sch.tables.has("users")); - }); +Run `npm run build:ts` - you should see errors because validation not wired yet. - it("should create a default schema without name", () => { - const sch = dumboSchema.schema({ - users: dumboSchema.table("users", { - columns: { - id: dumboSchema.column("id"), - }, - }), - }); +## Part 2: Implement Validation - assert.strictEqual(sch.schemaName, ""); - assert.strictEqual(sch.tables.size, 1); - }); +Update dumboDatabase function in dumboSchema.ts to add FK validation via conditional return types. - it("should create a named database", () => { - const db = dumboSchema.database("myapp", { - public: dumboSchema.schema("public", { - users: dumboSchema.table("users", { - columns: { - id: dumboSchema.column("id"), - }, - }), - }), - }); +1. Import ValidateDatabaseForeignKeys from '../foreignKeys' - assert.strictEqual(db.databaseName, "myapp"); - assert.strictEqual(db.schemas.size, 1); - assert.ok(db.schemas.has("public")); - }); +2. Find the TWO overload signatures that accept `schemas: Schemas` parameter (around lines 137 and 147) + - Do NOT modify overloads that accept single `schema` parameter - it("should handle DEFAULT_SCHEMA", () => { - const db = dumboSchema.database( - "myapp", - { - [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ - users: dumboSchema.table("users", { - columns: { - id: dumboSchema.column("id"), - }, - }), - }), - }, - { - defaultSchemaName: "main", - } - ); - - assert.strictEqual(db.databaseName, "myapp"); - assert.strictEqual(db.schemas.size, 1); - assert.ok(db.schemas.has("main")); - }); +3. Change their return type from: + ```typescript + DatabaseSchemaComponent + ``` - it("should create schema from table names", () => { - const sch = dumboSchema.schema.from("public", ["users", "posts"]); - assert.strictEqual(sch.schemaName, "public"); - assert.strictEqual(sch.tables.size, 2); - }); + To: + ```typescript + ValidateDatabaseForeignKeys> extends { valid: true } + ? DatabaseSchemaComponent + : ValidateDatabaseForeignKeys> + ``` - it("should create database from schema names", () => { - const db = dumboSchema.database.from("myapp", ["public", "analytics"]); - assert.strictEqual(db.databaseName, "myapp"); - assert.strictEqual(db.schemas.size, 2); - }); +4. Update the implementation function's return statement (around line 197) to cast the result: + ```typescript + return databaseSchemaComponent({ + databaseName, + schemas: schemaMap as Schemas, + ...dbOptions, + }) as ValidateDatabaseForeignKeys< + DatabaseSchemaComponent + > extends { valid: true } + ? DatabaseSchemaComponent + : ValidateDatabaseForeignKeys>; + ``` + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Conditional return type added to database() overloads that accept schemas +- Type errors appear AT THE database() CALL SITE when invalid FKs provided +- Error messages show which FK validation failed +- Valid schemas continue to work with no changes +- All quality gates pass + +--- + +### Step 20: Wire Validation to schema() Function + +**Context:** Add FK validation to schema() for foreign keys that reference tables within the same schema. Cross-schema references cannot be validated at this level. + +**Implementation Strategy:** + +Similar to database(), use conditional return types on `schema()` function overloads. However, validation is limited to: +- FK columns must exist in the table +- FK references must be within the SAME schema (format: `schema.table.column` where schema matches current schema name) +- Cross-schema references will only be validated at database() level + +**Prompt:** +``` + +IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. + +## Part 1: Add Failing Type Tests + +Add to foreignKeyValidation.type.spec.ts: + +```typescript +// TEST: Invalid intra-schema FK at schema() level +const _schemaInvalidFK = schema('public', { + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['invalid_col'], references: ['public.users.id'] }, + ], + }), +}); + +type _SchemaInvalidResult = typeof _schemaInvalidFK; +type _Test_SchemaInvalid = Expect< + Equal<_SchemaInvalidResult, DatabaseSchemaSchemaComponent> +>; // Should FAIL + +// TEST: Valid intra-schema FK +const _schemaValidFK = schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + }), }); + +type _SchemaValidResult = typeof _schemaValidFK; +type _Test_SchemaValid = Expect< + Equal<_SchemaValidResult, DatabaseSchemaSchemaComponent> +>; // Should PASS +``` + +Run `npm run build:ts` - should see errors because schema validation not wired yet. + +## Part 2: Implement Validation + +Update dumboDatabaseSchema function in dumboSchema.ts to add FK validation for intra-schema references. + +1. Import ValidateSchemaForeignKeys from '../foreignKeys' + +2. Find the TWO overload signatures that accept `tables: Tables` parameter + +3. Change their return type from: + ```typescript + DatabaseSchemaSchemaComponent + ``` + + To: + ```typescript + ValidateSchemaForeignKeys< + DatabaseSchemaSchemaComponent, + AllColumnReferences> + > extends { valid: true } + ? DatabaseSchemaSchemaComponent + : ValidateSchemaForeignKeys< + DatabaseSchemaSchemaComponent, + AllColumnReferences> + > + ``` + +4. Update the implementation function's return statement to cast the result similarly + +Note: Schema-level validation only catches intra-schema FK errors. Cross-schema FKs are validated at database() level (Step 19). + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + +``` + +**Acceptance Criteria:** +- Conditional return type added to schema() overloads +- Intra-schema FK errors caught at schema() call site +- Cross-schema FKs pass schema validation (caught later at database level) +- Database-level validation still primary mechanism +- All quality gates pass + +--- + +### Step 21: Add Real-World Test Cases + +**Context:** Test the validation with real schema definitions in dumboSchema.unit.spec.ts. + +**Prompt:** +``` + +Add real-world test cases to dumboSchema.unit.spec.ts demonstrating FK validation. + +Add a new describe block 'Foreign Key Validation' with test cases for: + +- Valid single FK +- Valid composite FK +- Self-referential FK +- Multiple FKs in one table +- Cross-schema FK + +Each test should verify the FK data is stored correctly in the schema component. + +After implementation, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit + +All must pass before proceeding. + ``` -**Validation**: +**Acceptance Criteria:** +- All real-world test cases added +- Tests verify FK data is stored correctly +- Tests demonstrate various FK scenarios +- All quality gates pass + +--- + +### Step 22: Add Comprehensive Compile-Time Validation Tests -- [ ] All tests pass -- [ ] Tests cover main functionality +**Context:** Create type-level tests that verify all FK validation scenarios work at compile time. -### Step 5: Usage Example +**Prompt:** +``` + +Add comprehensive type-level validation tests to foreignKeyValidation.type.spec.ts (after the basic type utility tests). -**File**: `/src/packages/dumbo/src/core/schema/dumboSchema/README.md` +Add these test cases demonstrating compile-time FK validation: ```typescript -import { dumboSchema } from "@event-driven-io/dumbo"; - -// Simple database with tables in default schema -const simpleDb = dumboSchema.database( - "myapp", - { - [dumboSchema.DEFAULT_SCHEMA]: dumboSchema.schema({ - users: dumboSchema.table("users", { - columns: { - id: dumboSchema.column("id"), - email: dumboSchema.column("email"), - name: dumboSchema.column("name"), - }, - indexes: { - idx_email: dumboSchema.index("idx_email", ["email"], { - unique: true, - }), +// ============================================================================ +// COMPILE-TIME FK VALIDATION TESTS +// ============================================================================ + +import { dumboSchema } from '../dumboSchema/dumboSchema'; +const { database, schema, table, column } = dumboSchema; + +// TEST 1: Invalid column name - should be error type +const _dbInvalidColumn = database('test', { + public: schema('public', { + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['nonexistent_col'], references: ['public.users.id'] }, + ], + }), + }), +}); + +type _InvalidColType = typeof _dbInvalidColumn; +// When validation fails, type should NOT equal DatabaseSchemaComponent +type _Test_InvalidColumn = Expect< + Equal<_InvalidColType, DatabaseSchemaComponent> +> extends true + ? never + : true; // Should be true (types don't match) + +// TEST 2: Invalid reference path - should be error type +const _dbInvalidRef = database('test', { + public: schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['user_id'], references: ['public.nonexistent.id'] }, + ], + }), + }), +}); + +type _InvalidRefType = typeof _dbInvalidRef; +type _Test_InvalidRef = Expect< + Equal<_InvalidRefType, DatabaseSchemaComponent> +> extends true + ? never + : true; // Should be true + +// TEST 3: Length mismatch - should be error type +const _dbLengthMismatch = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['user_id', 'tenant_id'], references: ['public.users.id'] }, + ], + }), + }), +}); + +type _LengthMismatchType = typeof _dbLengthMismatch; +type _Test_LengthMismatch = Expect< + Equal<_LengthMismatchType, DatabaseSchemaComponent> +> extends true + ? never + : true; // Should be true + +// TEST 4: Valid FK - should work perfectly (type matches) +const _dbValidFK = database('test', { + public: schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + }), + }), +}); + +type _ValidFKType = typeof _dbValidFK; +type _Test_ValidFK = Expect< + Equal<_ValidFKType, DatabaseSchemaComponent> +>; // Should PASS (types match) + +// TEST 5: Composite FK - should work +const _dbCompositeFK = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + foreignKeys: [ + { + columns: ['user_id', 'tenant_id'], + references: ['public.users.id', 'public.users.tenant_id'], }, - }), + ], }), - }, - { - defaultSchemaName: "public", // PostgreSQL default - } -); - -// Database with multiple schemas -const multiSchemaDb = dumboSchema.database("myapp", { - public: dumboSchema.schema("public", { - users: dumboSchema.table("users", { + }), +}); + +type _CompositeFKType = typeof _dbCompositeFK; +type _Test_CompositeFK = Expect< + Equal<_CompositeFKType, DatabaseSchemaComponent> +>; // Should PASS + +// TEST 6: Self-referential FK - should work +const _dbSelfRef = database('test', { + public: schema('public', { + users: table('users', { columns: { - id: dumboSchema.column("id"), - email: dumboSchema.column("email"), + id: column('id', Varchar('max')), + manager_id: column('manager_id', Varchar('max')), }, + foreignKeys: [ + { columns: ['manager_id'], references: ['public.users.id'] }, + ], + }), + }), +}); + +type _SelfRefType = typeof _dbSelfRef; +type _Test_SelfRef = Expect< + Equal<_SelfRefType, DatabaseSchemaComponent> +>; // Should PASS + +// TEST 7: Cross-schema FK - should work +const _dbCrossSchema = database('test', { + public: schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, }), }), - analytics: dumboSchema.schema("analytics", { - events: dumboSchema.table("events", { + analytics: schema('analytics', { + events: table('events', { columns: { - id: dumboSchema.column("id"), - user_id: dumboSchema.column("user_id"), - timestamp: dumboSchema.column("timestamp"), + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), }, + foreignKeys: [ + { columns: ['user_id'], references: ['public.users.id'] }, + ], }), }), }); -// Access using name-based maps -const publicSchema = multiSchemaDb.schemas.get("public"); -const usersTable = publicSchema?.tables.get("users"); -const emailColumn = usersTable?.columns.get("email"); +type _CrossSchemaType = typeof _dbCrossSchema; +type _Test_CrossSchema = Expect< + Equal<_CrossSchemaType, DatabaseSchemaComponent> +>; // Should PASS ``` -## Execution Plan +After adding tests, run quality gates as subagents: + +1. npm run fix +2. npm run build:ts +3. npm run test:unit -### Setup Subagents +All must pass. The invalid FK tests (1-3) should show type errors but still compile (they're type-level tests). -1. **Build Agent**: `npm run build:ts:watch` from `/home/oskar/Repos/Pongo/src` -2. **Linter Agent**: `npm run fix` on demand from `/home/oskar/Repos/Pongo/src` -3. **Test Agent**: `npm run test:unit` from `/home/oskar/Repos/Pongo/src` +Also verify the existing multiSchemaDb example (around line 163 in dumboSchema.unit.spec.ts) still works. -### Implementation Process +``` + +**Acceptance Criteria:** +- All 7 type-level test scenarios added +- Invalid FKs (tests 1-3) show they DON'T match DatabaseSchemaComponent type +- Valid FKs (tests 4-7) show they DO match DatabaseSchemaComponent type +- Existing multiSchemaDb example compiles without errors +- All quality gates pass -1. Start all subagents -2. Create constants.ts → Check compilation → Run linter → **Ask for confirmation** ✅ -3. Create index.ts → Check compilation → Run linter → **Ask for confirmation** ✅ -4. Update exports → Check compilation → Run linter → **Ask for confirmation** ✅ -5. Create tests → Ensure tests pass → **Ask for confirmation** ✅ -6. Don't commit after step. User will do it when confirming. +--- -## Success Criteria +## Summary -1. ✅ Code compiles -2. ✅ Linter passes -3. ✅ Unit tests pass -4. ✅ Fix issues, don't just report them +This implementation plan provides 22 detailed, step-by-step prompts to implement type-safe foreign key validation in the Dumbo schema system. Each step: -## Questions to Ask If Unsure +- Follows TDD principles (tests first, implementation second) +- Builds incrementally on previous work +- Has clear acceptance criteria +- Runs quality gates (fix, build, test) as subagents after each step +- Is self-contained and executable +- No orphaned code - everything integrates -- Should column/index builders accept additional metadata beyond what's in the current components? Not for now. -- Should we validate that DEFAULT_SCHEMA isn't used as an actual schema name? Yes. -- Should the default database name be configurable or just use 'database'? Same as schema +The final result will be: +- ✅ Compile-time validation of foreign keys +- ✅ No `as const` required +- ✅ Support for composite and self-referential FKs +- ✅ Clear, actionable error messages +- ✅ Fully tested with type tests and unit tests + +Total estimated implementation time: 4-6 hours for an experienced TypeScript developer. +``` diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts index d79527c6..f5eb61d1 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts @@ -36,6 +36,10 @@ export type DatabaseSchemaComponent< }> >; +export type AnyDatabaseSchemaComponent = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + DatabaseSchemaComponent; + export const databaseSchemaComponent = < Schemas extends DatabaseSchemas = DatabaseSchemas, >({ diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts new file mode 100644 index 00000000..efcc11a2 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts @@ -0,0 +1,68 @@ +import type { + AnyDatabaseSchemaSchemaComponent, + AnyTableSchemaComponent, + DatabaseSchemaComponent, + DatabaseSchemas, + DatabaseSchemaSchemaComponent, + DatabaseSchemaTables, + TableColumnNames, + TableColumns, + TableSchemaComponent, +} from '../'; + +export type ExtractSchemaNames = + DB extends DatabaseSchemaComponent + ? keyof Schemas & string + : never; + +export type ExtractTableNames = + Schema extends DatabaseSchemaSchemaComponent< + infer Tables extends DatabaseSchemaTables + > + ? keyof Tables & string + : never; + +export type ExtractColumnNames
= + Table extends TableSchemaComponent + ? TableColumnNames> + : never; + +export type AllColumnReferences = + DB extends DatabaseSchemaComponent + ? { + [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< + infer Tables + > + ? { + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns + > + ? { + [ColumnName in keyof Columns]: `${SchemaName & + string}.${TableName & string}.${ColumnName & string}`; + }[keyof Columns] + : never; + }[keyof Tables] + : never; + }[keyof Schemas] + : never; + +export type ForeignKeyDefinition = { + readonly columns: readonly Columns[]; + readonly references: readonly References[]; +}; + +export const foreignKey = < + const Columns extends readonly string[], + const References extends readonly string[], +>( + columns: Columns, + references: References, +) => { + return { + columns, + references, + } as const; +}; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyForeignKeyDefinition = ForeignKeyDefinition; diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts new file mode 100644 index 00000000..2d148abf --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts @@ -0,0 +1,186 @@ +import type { + AnyDatabaseSchemaComponent, + AnyDatabaseSchemaSchemaComponent, + DatabaseSchemaComponent, + DatabaseSchemaSchemaComponent, +} from '../'; +import type { + AnyTableSchemaComponent, + TableSchemaComponent, +} from '../tableSchemaComponent'; +import type { TableColumnNames } from '../tableTypesInference'; +import type { + AllColumnReferences, + AnyForeignKeyDefinition, +} from './foreignKeyTypes'; + +export type ValidationResult< + Valid extends boolean, + Error = never, +> = Valid extends true ? { valid: true } : { valid: false; error: Error }; + +type GetArrayLength = T['length']; + +export type ValidateForeignKeyLength< + FK extends { columns: readonly unknown[]; references: readonly unknown[] }, +> = + GetArrayLength extends GetArrayLength + ? ValidationResult + : ValidationResult< + false, + `Foreign key columns and references must have the same length. Got ${GetArrayLength} columns and ${GetArrayLength} references.` + >; + +type FindInvalidColumns< + Columns extends readonly string[], + ValidColumns extends string, + Invalid extends string[] = [], +> = Columns extends readonly [infer First, ...infer Rest] + ? First extends string + ? Rest extends readonly string[] + ? First extends ValidColumns + ? FindInvalidColumns + : FindInvalidColumns + : Invalid + : Invalid + : Invalid; + +type AllInTuple< + Tuple extends readonly string[], + Union extends string, +> = Tuple extends readonly [infer First, ...infer Rest] + ? First extends Union + ? Rest extends readonly string[] + ? AllInTuple + : true + : false + : true; + +export type ValidateForeignKeyColumns< + FK extends { columns: readonly string[] }, + ValidColumns extends string, +> = + AllInTuple extends true + ? ValidationResult + : ValidationResult< + false, + `Invalid foreign key columns: ${FindInvalidColumns extends infer Invalid ? (Invalid extends string[] ? Invalid[number] : never) : never}. Available columns: ${ValidColumns}` + >; + +type FindInvalidReferences< + References extends readonly string[], + ValidReferences extends string, + Invalid extends string[] = [], +> = References extends readonly [infer First, ...infer Rest] + ? First extends string + ? Rest extends readonly string[] + ? First extends ValidReferences + ? FindInvalidReferences + : FindInvalidReferences + : Invalid + : Invalid + : Invalid; + +export type ValidateForeignKeyReferences< + FK extends { references: readonly string[] }, + ValidReferences extends string, +> = + AllInTuple extends true + ? ValidationResult + : ValidationResult< + false, + `Invalid foreign key references: ${FindInvalidReferences extends infer Invalid ? (Invalid extends string[] ? Invalid[number] : never) : never}. Available references: ${ValidReferences}` + >; + +export type ValidateSingleForeignKey< + FK extends { columns: readonly string[]; references: readonly string[] }, + TableColumns extends string, + ValidReferences extends string, +> = + ValidateForeignKeyLength extends { valid: false; error: infer E } + ? ValidationResult + : ValidateForeignKeyColumns extends { + valid: false; + error: infer E; + } + ? ValidationResult + : ValidateForeignKeyReferences extends { + valid: false; + error: infer E; + } + ? ValidationResult + : ValidationResult; + +export type ValidateForeignKeyArray< + FKs extends readonly AnyForeignKeyDefinition[], + TableColumns extends string, + ValidReferences extends string, +> = FKs extends readonly [] + ? ValidationResult + : ValidateSingleForeignKey< + FKs[number], + TableColumns, + ValidReferences + > extends { + valid: false; + error: infer E; + } + ? ValidationResult + : ValidationResult; + +export type ValidateTableForeignKeys< + Table extends AnyTableSchemaComponent, + ValidReferences extends string, +> = + Table extends TableSchemaComponent + ? ValidateForeignKeyArray< + FKs, + TableColumnNames
& string, + ValidReferences + > + : ValidationResult; + +export type ValidateTablesInSchema< + Tables extends Record, + ValidReferences extends string, +> = { + [TableName in keyof Tables]: ValidateTableForeignKeys< + Tables[TableName], + ValidReferences + >; +}[keyof Tables] extends infer Results + ? Results extends { valid: true } + ? ValidationResult + : Results extends { valid: false; error: infer E } + ? ValidationResult + : ValidationResult + : ValidationResult; + +export type ValidateSchemaForeignKeys< + Schema extends AnyDatabaseSchemaSchemaComponent, + ValidReferences extends string, +> = + Schema extends DatabaseSchemaSchemaComponent + ? ValidateTablesInSchema + : ValidationResult; + +export type ValidateSchemasInDatabase< + Schemas extends Record, + ValidReferences extends string, +> = { + [SchemaName in keyof Schemas]: ValidateSchemaForeignKeys< + Schemas[SchemaName], + ValidReferences + >; +}[keyof Schemas] extends infer Results + ? Results extends { valid: true } + ? ValidationResult + : Results extends { valid: false; error: infer E } + ? ValidationResult + : ValidationResult + : ValidationResult; + +export type ValidateDatabaseForeignKeys = + DB extends DatabaseSchemaComponent + ? ValidateSchemasInDatabase> + : ValidationResult; diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts new file mode 100644 index 00000000..4ec3283a --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts @@ -0,0 +1,572 @@ +import type { + AnyColumnSchemaComponent, + AnyDatabaseSchemaComponent, + AnyDatabaseSchemaSchemaComponent, + AnyTableSchemaComponent, + DatabaseSchemaComponent, + DatabaseSchemaSchemaComponent, + TableSchemaComponent, +} from '../'; +import { SQL } from '../../../sql'; +import type { Equal, Expect } from '../../../testing'; +import { dumboSchema } from '../../dumboSchema'; +import type { + AllColumnReferences, + ExtractColumnNames, + ExtractSchemaNames, + ExtractTableNames, + ForeignKeyDefinition, +} from './foreignKeyTypes'; + +const { database, schema, table, column } = dumboSchema; +const { Varchar } = SQL.column.type; + +type _DB1 = DatabaseSchemaComponent<{ + public: AnyDatabaseSchemaSchemaComponent; +}>; +type _Test1 = Expect, 'public'>>; + +type _DB2 = DatabaseSchemaComponent<{ + public: AnyDatabaseSchemaSchemaComponent; + analytics: AnyDatabaseSchemaSchemaComponent; +}>; +type _Test2 = Expect, 'public' | 'analytics'>>; + +type _Schema1 = DatabaseSchemaSchemaComponent<{ + users: AnyTableSchemaComponent; +}>; +type _Test3 = Expect, 'users'>>; + +type _Schema2 = DatabaseSchemaSchemaComponent<{ + users: AnyTableSchemaComponent; + posts: AnyTableSchemaComponent; + comments: AnyTableSchemaComponent; +}>; +type _Test4 = Expect< + Equal, 'users' | 'posts' | 'comments'> +>; + +type _Table1 = TableSchemaComponent<{ + id: AnyColumnSchemaComponent; +}>; +type _Test5 = Expect, 'id'>>; + +type _Table2 = TableSchemaComponent<{ + id: AnyColumnSchemaComponent; + email: AnyColumnSchemaComponent; + name: AnyColumnSchemaComponent; + created_at: AnyColumnSchemaComponent; +}>; +type _Test6 = Expect< + Equal, 'id' | 'email' | 'name' | 'created_at'> +>; + +const _db1 = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + }, + }), + }), +}); + +type _Result1 = AllColumnReferences; +type _Test7 = Expect>; + +const _db2 = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + title: column('title', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + }), + }), +}); + +type _Result2 = AllColumnReferences; +type _Test8 = Expect< + Equal< + _Result2, + | 'public.users.id' + | 'public.users.email' + | 'public.posts.id' + | 'public.posts.title' + | 'public.posts.user_id' + > +>; + +const _db3 = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + }, + }), + }), + analytics: schema('analytics', { + events: table('events', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + event_type: column('event_type', Varchar('max')), + }, + }), + }), +}); + +type _Result3 = AllColumnReferences; +type _Test9 = Expect< + Equal< + _Result3, + | 'public.users.id' + | 'public.users.email' + | 'analytics.events.id' + | 'analytics.events.user_id' + | 'analytics.events.event_type' + > +>; + +const _validFK: ForeignKeyDefinition = { + columns: ['user_id'], + references: ['public.users.id'], +}; + +type _ColumnsType = typeof _validFK.columns; +type _Test10 = Expect>; + +const _compositeFK: ForeignKeyDefinition = { + columns: ['user_id', 'tenant_id'], + references: ['public.users.id', 'public.users.tenant_id'], +}; + +type _CompositeColumnsType = typeof _compositeFK.columns; +type _CompositeReferencesType = typeof _compositeFK.references; +type _Test11 = Expect>; +type _Test12 = Expect>; + +import type { IsError } from '../../../testing/typesTesting'; +import type { + ValidateDatabaseForeignKeys, + ValidateForeignKeyLength, +} from './foreignKeyValidation'; + +type _FK_LengthMismatch = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; +}; + +type _Result_LengthMismatch = ValidateForeignKeyLength<_FK_LengthMismatch>; +type _Test13 = Expect>; + +type _FK_SingleMatch = { + columns: ['user_id']; + references: ['public.users.id']; +}; + +type _FK_CompositeMatch = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id', 'public.users.tenant_id']; +}; + +type _Result_SingleMatch = ValidateForeignKeyLength<_FK_SingleMatch>; +type _Result_CompositeMatch = ValidateForeignKeyLength<_FK_CompositeMatch>; +type _Test14 = Expect>; +type _Test15 = Expect>; + +import type { ValidateForeignKeyColumns } from './foreignKeyValidation'; + +type _FK_InvalidColumn = { + columns: ['user_id', 'invalid_col']; + references: ['public.users.id', 'public.users.tenant_id']; +}; + +type _Result_InvalidColumn = ValidateForeignKeyColumns< + _FK_InvalidColumn, + 'id' | 'email' | 'user_id' +>; +type _Test16 = Expect>; + +type _FK_ValidColumns = { + columns: ['user_id']; + references: ['public.users.id']; +}; + +type _FK_ValidCompositeColumns = { + columns: ['user_id', 'email']; + references: ['public.users.id', 'public.users.email']; +}; + +type _Result_ValidColumns = ValidateForeignKeyColumns< + _FK_ValidColumns, + 'id' | 'email' | 'user_id' +>; +type _Result_ValidCompositeColumns = ValidateForeignKeyColumns< + _FK_ValidCompositeColumns, + 'id' | 'email' | 'user_id' +>; +type _Test17 = Expect>; +type _Test18 = Expect>; + +import type { ValidateForeignKeyReferences } from './foreignKeyValidation'; + +type _FK_InvalidReference = { + columns: ['user_id']; + references: ['public.nonexistent.id']; +}; + +type _Result_InvalidReference = ValidateForeignKeyReferences< + _FK_InvalidReference, + 'public.users.id' | 'public.users.email' | 'public.posts.id' +>; +type _Test19 = Expect>; + +type _FK_ValidReference = { + columns: ['user_id']; + references: ['public.users.id']; +}; + +type _FK_ValidCompositeReference = { + columns: ['user_id', 'post_id']; + references: ['public.users.id', 'public.posts.id']; +}; + +type _Result_ValidReference = ValidateForeignKeyReferences< + _FK_ValidReference, + 'public.users.id' | 'public.users.email' | 'public.posts.id' +>; +type _Result_ValidCompositeReference = ValidateForeignKeyReferences< + _FK_ValidCompositeReference, + 'public.users.id' | 'public.users.email' | 'public.posts.id' +>; +type _Test20 = Expect>; +type _Test21 = Expect>; + +import type { ValidateSingleForeignKey } from './foreignKeyValidation'; + +type _FK_Complete_Valid = { + columns: ['user_id']; + references: ['public.users.id']; +}; + +type _Result_Complete_Valid = ValidateSingleForeignKey< + _FK_Complete_Valid, + 'id' | 'user_id', + 'public.users.id' | 'public.users.email' +>; +type _Test22 = Expect>; + +type _FK_Complete_LengthError = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; +}; + +type _Result_Complete_LengthError = ValidateSingleForeignKey< + _FK_Complete_LengthError, + 'id' | 'user_id' | 'tenant_id', + 'public.users.id' | 'public.users.email' +>; +type _Test23 = Expect>; + +type _FK_Complete_ColumnError = { + columns: ['invalid_col']; + references: ['public.users.id']; +}; + +type _Result_Complete_ColumnError = ValidateSingleForeignKey< + _FK_Complete_ColumnError, + 'id' | 'user_id', + 'public.users.id' | 'public.users.email' +>; +type _Test24 = Expect>; + +type _FK_Complete_ReferenceError = { + columns: ['user_id']; + references: ['public.invalid.id']; +}; + +type _Result_Complete_ReferenceError = ValidateSingleForeignKey< + _FK_Complete_ReferenceError, + 'id' | 'user_id', + 'public.users.id' | 'public.users.email' +>; +type _Test24A = Expect< + Equal< + _Result_Complete_ReferenceError, + { + valid: false; + error: + | 'Invalid foreign key references: public.invalid.id. Available references: public.users.id' + | 'Invalid foreign key references: public.invalid.id. Available references: public.users.email'; + } + > +>; + +import type { ValidateForeignKeyArray } from './foreignKeyValidation'; + +type _FKArray_Mixed = readonly [ + { + columns: ['user_id']; + references: ['public.users.id']; + }, + { + columns: ['invalid_col']; + references: ['public.users.email']; + }, +]; + +type _Result_FKArray_Mixed = ValidateForeignKeyArray< + _FKArray_Mixed, + 'id' | 'user_id', + 'public.users.id' | 'public.users.email' +>; + +type _Test25A = Expect< + Equal< + _Result_FKArray_Mixed, + { + valid: false; + error: + | 'Invalid foreign key columns: invalid_col. Available columns: user_id' + | 'Invalid foreign key columns: invalid_col. Available columns: id'; + } + > +>; +type _Test35 = Expect>; + +type _FKArray_AllValid = readonly [ + { + columns: ['user_id']; + references: ['public.users.id']; + }, + { + columns: ['email']; + references: ['public.users.email']; + }, +]; + +type _Result_FKArray_AllValid = ValidateForeignKeyArray< + _FKArray_AllValid, + 'id' | 'user_id' | 'email', + 'public.users.id' | 'public.users.email' +>; +type _ValidateForeignKeyArrayResult_InvalidFK = ValidateForeignKeyArray< + [{ columns: ['invalid']; references: ['public.users.id'] }], + 'id' | 'user_id', + 'public.users.id' | 'public.users.email' +>; +type _TestValidateForeignKeyArrayResult_InvalidFK = Expect< + IsError<_ValidateForeignKeyArrayResult_InvalidFK> +>; + +import type { ValidateTableForeignKeys } from './foreignKeyValidation'; + +type _Table_NoFKs = TableSchemaComponent<{ + id: AnyColumnSchemaComponent; + email: AnyColumnSchemaComponent; +}>; + +type _Result_NoFKs = ValidateTableForeignKeys< + _Table_NoFKs, + 'public.users.id' | 'public.users.email' +>; +type _Test26 = Expect>; + +type _Table_SingleFK = TableSchemaComponent< + { + id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + }, + [{ columns: ['user_id']; references: ['public.users.id'] }] +>; + +type _Result_SingleFK = ValidateTableForeignKeys< + _Table_SingleFK, + 'public.users.id' | 'public.users.email' +>; +type _Test27 = Expect>; + +type _Table_MultipleFK = TableSchemaComponent< + { + id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + author_id: AnyColumnSchemaComponent; + }, + [ + { columns: ['user_id']; references: ['public.users.id'] }, + { columns: ['author_id']; references: ['public.users.id'] }, + ] +>; + +type _Result_MultipleFK = ValidateTableForeignKeys< + _Table_MultipleFK, + 'public.users.id' | 'public.users.email' +>; +type _Test28 = Expect>; + +type _Table_InvalidFK = TableSchemaComponent< + { + id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + }, + [{ readonly columns: ['id']; references: readonly ['public.users.id'] }] +>; + +type _Result_InvalidFK = ValidateTableForeignKeys< + _Table_InvalidFK, + 'public.posts.id' | 'public.users.email' +>; +type _Test29 = Expect>; + +import type { ValidateSchemaForeignKeys } from './foreignKeyValidation'; + +type _Schema_MultiTable = DatabaseSchemaSchemaComponent<{ + users: TableSchemaComponent<{ + id: AnyColumnSchemaComponent; + email: AnyColumnSchemaComponent; + }>; + posts: TableSchemaComponent< + { + id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + }, + [{ columns: ['user_id']; references: ['public.users.id'] }] + >; +}>; + +type _Result_Schema_Valid = ValidateSchemaForeignKeys< + _Schema_MultiTable, + 'public.users.id' | 'public.users.email' | 'public.posts.id' +>; +type _Test30 = Expect>; + +type _Schema_WithError = DatabaseSchemaSchemaComponent<{ + posts: TableSchemaComponent< + { + id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + }, + [{ columns: ['id']; references: ['public.users.id'] }] + >; +}>; + +type _Result_Schema_Error = ValidateSchemaForeignKeys< + _Schema_WithError, + 'public.posts.id' | 'public.users.email' +>; +type _Test31 = Expect>; + +const _dbWithErrorVSInDB = database('test', { + public: schema('public', { + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + }), + }), +}); + +type _Test_ValidateSchemasInDatabaseResult_DbError = Expect< + IsError +>; + +const _fullDb = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + }), + }), +}); + +type _Result_FullDb = ValidateDatabaseForeignKeys; +type _Test32 = Expect>; + +const _dbWithSelfRef = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + manager_id: column('manager_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['manager_id'], references: ['public.users.id'] }, + ], + }), + }), +}); + +type _Result_SelfRef = ValidateDatabaseForeignKeys; +type _Test33 = Expect>; + +const _dbWithError = database('test', { + public: schema('public', { + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + }), + }), +}); + +type _Test34 = Expect>; + +// TEST: Invalid column should cause type error at database() call +const _dbInvalidColumn = database('test', { + public: schema('public', { + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('id', Varchar('max')), + }, + foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + }), + }), +}); + +type _InvalidColResult = typeof _dbInvalidColumn; +type _Test_InvalidColumn = Expect>; + +// TEST: Valid FK should work +const _dbValid = database('test', { + public: schema('public', { + users: table('users', { + columns: { id: column('id', Varchar('max')) }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + }), + }), +}); + +type _ValidResult = typeof _dbValid; +type _Test_Valid = Expect< + Equal<_ValidResult extends AnyDatabaseSchemaComponent ? true : false, true> +>; // This should PASS diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts b/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts new file mode 100644 index 00000000..c99d8180 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts @@ -0,0 +1,3 @@ +export * from './foreignKeyTypes'; + +export * from './foreignKeyValidation'; diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts index 6aad254e..1ae8b8cb 100644 --- a/src/packages/dumbo/src/core/schema/components/index.ts +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -7,6 +7,7 @@ import { TableURN } from './tableSchemaComponent'; export * from './columnSchemaComponent'; export * from './databaseSchemaComponent'; export * from './databaseSchemaSchemaComponent'; +export * from './foreignKeys'; export * from './indexSchemaComponent'; export * from './tableSchemaComponent'; export * from './tableTypesInference'; diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 46f659e1..eba4d7fb 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -8,6 +8,7 @@ import { ColumnURNType, type AnyColumnSchemaComponent, } from './columnSchemaComponent'; +import type { ForeignKeyDefinition } from './foreignKeys/foreignKeyTypes'; import { IndexURNType, type IndexSchemaComponent, @@ -22,36 +23,59 @@ export const TableURN = ({ name }: { name: string }): TableURN => `${TableURNType}:${name}`; export type TableColumns = Record; +export type TableForeignKeys< + Columns extends string = string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + FKS extends ForeignKeyDefinition = ForeignKeyDefinition< + Columns, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any + >, +> = + FKS extends ForeignKeyDefinition + ? readonly ForeignKeyDefinition[] + : never; -export type TableSchemaComponent = - SchemaComponent< - TableURN, - Readonly<{ - tableName: string; - columns: ReadonlyMap & Columns; - primaryKey: TableColumnNames>[]; - indexes: ReadonlyMap; - addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; - addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; - }> - >; +export type TableSchemaComponent< + Columns extends TableColumns = TableColumns, + ForeignKeys extends TableForeignKeys< + keyof Columns & string + > = TableForeignKeys, +> = SchemaComponent< + TableURN, + Readonly<{ + tableName: string; + columns: ReadonlyMap & Columns; + primaryKey: TableColumnNames>[]; + foreignKeys: ForeignKeys; + indexes: ReadonlyMap; + addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; + addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; + }> +>; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyTableSchemaComponent = TableSchemaComponent; +export type AnyTableSchemaComponent = TableSchemaComponent; export const tableSchemaComponent = < Columns extends TableColumns = TableColumns, + const ForeignKeys extends TableForeignKeys = TableForeignKeys, >({ tableName, columns, primaryKey, + foreignKeys, ...migrationsOrComponents }: { tableName: string; columns?: Columns; - primaryKey?: TableColumnNames>[]; -} & SchemaComponentOptions): TableSchemaComponent => { + primaryKey?: TableColumnNames>[]; + foreignKeys?: ForeignKeys; +} & SchemaComponentOptions): TableSchemaComponent & { + foreignKeys: ForeignKeys; +} => { columns ??= {} as Columns; + foreignKeys ??= {} as ForeignKeys; const base = schemaComponent(TableURN({ name: tableName }), { migrations: migrationsOrComponents.migrations ?? [], @@ -65,6 +89,7 @@ export const tableSchemaComponent = < ...base, tableName, primaryKey: primaryKey ?? [], + foreignKeys, get columns() { const columnsMap = mapSchemaComponentsOfType( base.components, @@ -83,5 +108,7 @@ export const tableSchemaComponent = < }, addColumn: (column: AnyColumnSchemaComponent) => base.addComponent(column), addIndex: (index: IndexSchemaComponent) => base.addComponent(index), + } as TableSchemaComponent & { + foreignKeys: ForeignKeys; }; }; diff --git a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts index 20e482e1..78c9ca80 100644 --- a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts @@ -3,6 +3,14 @@ import type { AnyColumnSchemaComponent, ColumnSchemaComponent, } from './columnSchemaComponent'; +import type { + AnyDatabaseSchemaComponent, + DatabaseSchemaComponent, +} from './databaseSchemaComponent'; +import type { + AnyDatabaseSchemaSchemaComponent, + DatabaseSchemaSchemaComponent, +} from './databaseSchemaSchemaComponent'; import type { AnyTableSchemaComponent, TableColumns, @@ -39,3 +47,9 @@ export type TableRowType = T extends TableSchemaComponent ? InferTableRow : never; + +export type InferSchemaTables = + T extends DatabaseSchemaSchemaComponent ? Tables : never; + +export type InferDatabaseSchemas = + T extends DatabaseSchemaComponent ? Schemas : never; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index c41d1317..3c6be831 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,4 +1,5 @@ import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; +import type { ValidateDatabaseForeignKeys } from '../components'; import { type AnyDatabaseSchemaSchemaComponent, columnSchemaComponent, @@ -13,6 +14,7 @@ import { type IndexSchemaComponent, type TableColumnNames, type TableColumns, + type TableForeignKeys, tableSchemaComponent, type TableSchemaComponent, } from '../components'; @@ -57,15 +59,21 @@ const dumboIndex = ( ...options, }); -const dumboTable = ( +const dumboTable = < + Columns extends TableColumns = TableColumns, + const ForeignKeys extends TableForeignKeys< + keyof Columns & string + > = TableForeignKeys, +>( name: string, definition: { columns?: Columns; - primaryKey?: TableColumnNames>[]; + primaryKey?: TableColumnNames>[]; + foreignKeys?: ForeignKeys; indexes?: Record; } & SchemaComponentOptions, -): TableSchemaComponent => { - const { columns, indexes, primaryKey, ...options } = definition; +): TableSchemaComponent => { + const { columns, indexes, primaryKey, foreignKeys, ...options } = definition; const components = [...(indexes ? Object.values(indexes) : [])]; @@ -73,6 +81,7 @@ const dumboTable = ( tableName: name, columns: columns ?? ({} as Columns), primaryKey: primaryKey ?? [], + ...(foreignKeys !== undefined ? { foreignKeys } : {}), components, ...options, }); @@ -126,22 +135,31 @@ dumboDatabaseSchema.from = ( : dumboDatabaseSchema(tables); }; +type ValidatedDatabaseSchemaComponent< + Schemas extends DatabaseSchemas = DatabaseSchemas, +> = + ValidateDatabaseForeignKeys> extends { + valid: true; + } + ? DatabaseSchemaComponent + : ValidateDatabaseForeignKeys>; + function dumboDatabase( schemas: Schemas, -): DatabaseSchemaComponent; +): ValidatedDatabaseSchemaComponent; function dumboDatabase( schema: DatabaseSchemaSchemaComponent, -): DatabaseSchemaComponent; +): ValidatedDatabaseSchemaComponent; function dumboDatabase( databaseName: string, schemas: Schemas, options?: SchemaComponentOptions, -): DatabaseSchemaComponent; +): ValidatedDatabaseSchemaComponent; function dumboDatabase( databaseName: string, schema: AnyDatabaseSchemaSchemaComponent, options?: SchemaComponentOptions, -): DatabaseSchemaComponent; +): ValidatedDatabaseSchemaComponent; function dumboDatabase( nameOrSchemas: string | DatabaseSchemaSchemaComponent | Schemas, schemasOrOptions?: @@ -149,7 +167,7 @@ function dumboDatabase( | Schemas | SchemaComponentOptions, options?: SchemaComponentOptions, -): DatabaseSchemaComponent { +): ValidatedDatabaseSchemaComponent { const databaseName = typeof nameOrSchemas === 'string' ? nameOrSchemas : DEFAULT_DATABASE_NAME; @@ -184,7 +202,7 @@ function dumboDatabase( dumboDatabase.from = ( databaseName: string | undefined, schemaNames: string[], -): DatabaseSchemaComponent => { +): ValidatedDatabaseSchemaComponent => { const schemas = schemaNames.reduce( (acc, schemaName) => { acc[schemaName] = dumboDatabaseSchema( diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index ac2293da..57e6ac4e 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -3,6 +3,7 @@ import { describe, it } from 'node:test'; import { SQL } from '../../sql'; import type { Equal, Expect } from '../../testing'; import type { TableColumnNames, TableRowType } from '../components'; +import { foreignKey } from '../components'; import { dumboSchema } from './index'; const { database, schema, table, column, index } = dumboSchema; @@ -150,6 +151,21 @@ const users = table('users', { email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, + foreignKeys: [foreignKey(['id'], ['public.profiles.user_id'])], +}); + +const _users2 = table('users', { + columns: { + id: column('id', Varchar('max'), { primaryKey: true, notNull: true }), + email: column('email', Varchar('max'), { notNull: true }), + name: column('name', Varchar('max')), + }, + foreignKeys: [ + { + columns: ['id'], + references: ['public.profiles.user_id'], + }, + ], }); export const simpleDb = database( @@ -179,6 +195,12 @@ const multiSchemaDb = database('myapp', { userId: column('user_id', Varchar('max')), timestamp: column('timestamp', Varchar('max')), }, + foreignKeys: [ + { + columns: ['userId'], + references: ['public.users.id'], + }, + ], }), }), }); @@ -199,3 +221,139 @@ type _MetadataColumnIsNullableObject = Expect< type UserColumns = TableColumnNames; const _userColumns: UserColumns[] = ['id', 'email', 'name', 'metadata']; + +void describe('Foreign Key Validation', () => { + void it('should accept valid single foreign key', () => { + const db = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + email: column('email', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['user_id'], references: ['public.users.id'] }, + ], + }), + }), + }); + + assert.ok(db.schemas.public.tables.posts.foreignKeys); + assert.deepStrictEqual( + db.schemas.public.tables.posts.foreignKeys[0].columns, + ['user_id'], + ); + }); + + void it('should accept valid composite foreign key', () => { + const db = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + tenant_id: column('tenant_id', Varchar('max')), + }, + foreignKeys: [ + { + columns: ['user_id', 'tenant_id'], + references: ['public.users.id', 'public.users.tenant_id'], + }, + ], + }), + }), + }); + + assert.deepStrictEqual( + db.schemas.public.tables.posts.foreignKeys[0].columns, + ['user_id', 'tenant_id'], + ); + }); + + void it('should accept self-referential foreign key', () => { + const db = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + manager_id: column('manager_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['manager_id'], references: ['public.users.id'] }, + ] as const, + }), + }), + }); + + assert.ok(db.schemas.public.tables.users.foreignKeys); + assert.deepStrictEqual( + db.schemas.public.tables.users.foreignKeys[0].references, + ['public.users.id'], + ); + }); + + void it('should accept multiple foreign keys in one table', () => { + const db = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + }, + }), + posts: table('posts', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + author_id: column('author_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['user_id'], references: ['public.users.id'] }, + { columns: ['author_id'], references: ['public.users.id'] }, + ] as const, + }), + }), + }); + + assert.strictEqual(db.schemas.public.tables.posts.foreignKeys.length, 2); + }); + + void it('should accept cross-schema foreign key', () => { + const db = database('test', { + public: schema('public', { + users: table('users', { + columns: { + id: column('id', Varchar('max')), + }, + }), + }), + analytics: schema('analytics', { + events: table('events', { + columns: { + id: column('id', Varchar('max')), + user_id: column('user_id', Varchar('max')), + }, + foreignKeys: [ + { columns: ['user_id'], references: ['public.users.id'] }, + ], + }), + }), + }); + + assert.deepStrictEqual( + db.schemas.analytics.tables.events.foreignKeys[0].references, + ['public.users.id'], + ); + }); +}); diff --git a/src/packages/dumbo/src/core/testing/typesTesting.ts b/src/packages/dumbo/src/core/testing/typesTesting.ts index b38f602e..b4a8a44a 100644 --- a/src/packages/dumbo/src/core/testing/typesTesting.ts +++ b/src/packages/dumbo/src/core/testing/typesTesting.ts @@ -3,3 +3,4 @@ export type Equal = (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 ? true : false; +export type IsError = T extends { valid: false } ? true : false; diff --git a/todo.md b/todo.md new file mode 100644 index 00000000..5a351423 --- /dev/null +++ b/todo.md @@ -0,0 +1,71 @@ +# Foreign Key Type Validation - Implementation Checklist + +## Phase 1: Foundation - Type Extraction Utilities +- [x] Step 1: Add ExpectError Helper +- [x] Step 2: Create Foreign Key Types File Structure +- [ ] Step 3: Implement ExtractSchemaNames Type Utility +- [ ] Step 4: Implement ExtractTableNames Type Utility +- [ ] Step 5: Implement ExtractColumnNames Type Utility + +## Phase 2: Column Reference Generation +- [ ] Step 6: Implement AllColumnReferences Type Utility (Single Schema) +- [ ] Step 7: Test AllColumnReferences with Multi-Schema Database + +## Phase 3: Foreign Key Type Definition +- [ ] Step 8: Define ForeignKeyDefinition Type +- [ ] Step 9: Update TableSchemaComponent to Include Foreign Keys +- [ ] Step 10: Update dumboTable to Accept Foreign Keys + +## Phase 4: Single Foreign Key Validation +- [ ] Step 11: Implement ValidateForeignKeyLength +- [ ] Step 12: Implement ValidateForeignKeyColumns +- [ ] Step 13: Implement ValidateForeignKeyReferences +- [ ] Step 14: Implement ValidateSingleForeignKey (Combine Validations) + +## Phase 5: Full Database Validation +- [ ] Step 15: Implement ValidateTableForeignKeys +- [ ] Step 16: Implement ValidateSchemaForeignKeys +- [ ] Step 17: Implement ValidateDatabaseForeignKeys + +## Phase 6: Integration with dumboSchema +- [ ] Step 18: Create Foreign Keys Barrel Export +- [ ] Step 19: Wire Validation to database() Function +- [ ] Step 20: Wire Validation to schema() Function + +## Phase 7: Real-World Testing +- [ ] Step 21: Add Real-World Test Cases +- [ ] Step 22: Update Existing Example in Test File + +--- + +## Quality Gates (Run after EVERY step as subagents) + +After each step is complete, you MUST run: + +1. ✓ `npm run fix` - Fix linting issues +2. ✓ `npm run build:ts` - Ensure TypeScript compiles +3. ✓ `npm run test:unit` - Run all tests + +**All three must pass before proceeding to the next step.** + +If any fail and cannot be fixed automatically, **STOP and ask for help**. + +--- + +## Progress Tracking + +**Current Step:** Not started + +**Completed Steps:** 0 / 22 + +**Estimated Time Remaining:** 4-6 hours + +--- + +## Notes + +- Follow TDD: Write type tests first, then implement +- Use `describe`/`it` for tests +- No unnecessary comments +- Keep code simple and maintainable +- Follow existing codebase patterns From 44c0589fb822634216421bc8a57b414a63350de5 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 25 Nov 2025 10:05:56 +0100 Subject: [PATCH 37/39] Renamed foreign keys into relationships --- plan.md | 409 +++++++++--------- .../core/schema/MIGRATION_UNIFICATION_PLAN.md | 2 +- .../schema/components/foreignKeys/index.ts | 3 - .../dumbo/src/core/schema/components/index.ts | 2 +- .../schema/components/relationships/index.ts | 3 + .../relationshipTypes.ts} | 8 +- .../relationshipValidation.ts} | 40 +- .../relationshipValidation.type.spec.ts} | 96 ++-- .../schema/components/tableSchemaComponent.ts | 40 +- .../core/schema/dumboSchema/dumboSchema.ts | 25 +- .../dumboSchema/dumboSchema.unit.spec.ts | 32 +- .../dumbo/src/core/schema/schema_feature.md | 2 +- todo.md | 23 +- 13 files changed, 356 insertions(+), 329 deletions(-) delete mode 100644 src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/index.ts rename src/packages/dumbo/src/core/schema/components/{foreignKeys/foreignKeyTypes.ts => relationships/relationshipTypes.ts} (90%) rename src/packages/dumbo/src/core/schema/components/{foreignKeys/foreignKeyValidation.ts => relationships/relationshipValidation.ts} (85%) rename src/packages/dumbo/src/core/schema/components/{foreignKeys/foreignKeyValidation.type.spec.ts => relationships/relationshipValidation.type.spec.ts} (82%) diff --git a/plan.md b/plan.md index f3f746d8..f07b9008 100644 --- a/plan.md +++ b/plan.md @@ -32,7 +32,7 @@ database() called → Extract all tables from each schema → Extract all columns from each table → Build union of all valid column references: 'schema.table.column' - → For each table with foreignKeys: + → For each table with relationships: → Validate each FK's columns array against table's columns → Validate each FK's references array against all valid column references → Validate columns.length === references.length @@ -94,8 +94,8 @@ Build the base type utilities to extract schema structure information. **Files to create/modify:** -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (new) -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (new) +- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (new) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (new) - `src/packages/dumbo/src/core/testing/typesTesting.ts` (modify - add ExpectError) **Deliverables:** @@ -111,8 +111,8 @@ Build utilities to generate all valid column reference paths. **Files to create/modify:** -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (modify) -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) **Deliverables:** @@ -125,14 +125,14 @@ Define the foreign key structure and add it to table schema component. **Files to create/modify:** -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (modify) - `src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts` (modify) -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) **Deliverables:** -- `ForeignKeyDefinition` type -- Updated `TableSchemaComponent` with generic FK parameter +- `RelationshipDefinition` type +- Updated `TableSchemaComponent` with generic FK parameter - Type tests for FK structure ### Phase 4: Single Foreign Key Validation @@ -141,15 +141,15 @@ Implement validation logic for a single foreign key definition. **Files to create/modify:** -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts` (new) -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts` (new) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) **Deliverables:** -- `ValidateForeignKeyColumns` - validate columns array -- `ValidateForeignKeyReferences` - validate references array -- `ValidateForeignKeyLength` - validate columns.length === references.length -- `ValidateSingleForeignKey` - combine all validations +- `ValidateRelationshipColumns` - validate columns array +- `ValidateRelationshipReferences` - validate references array +- `ValidateRelationshipLength` - validate columns.length === references.length +- `ValidateSingleRelationship` - combine all validations - Comprehensive error message types with helpful suggestions - Type tests for valid and invalid scenarios @@ -159,14 +159,14 @@ Implement database-level validation that checks all foreign keys across all tabl **Files to create/modify:** -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts` (modify) -- `src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts` (modify) +- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) **Deliverables:** -- `ValidateTableForeignKeys` - validate all FKs in a table -- `ValidateSchemaForeignKeys` - validate all FKs in a schema -- `ValidateDatabaseForeignKeys` - validate all FKs in entire database +- `ValidateTableRelationships` - validate all FKs in a table +- `ValidateSchemaRelationships` - validate all FKs in a schema +- `ValidateDatabaseRelationships` - validate all FKs in entire database - Type tests for multi-table, multi-schema validation ### Phase 6: Integration with dumboSchema @@ -177,11 +177,11 @@ Wire up validation to the `database()` and `schema()` functions. - `src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts` (modify) - `src/packages/dumbo/src/core/schema/components/index.ts` (modify - re-export FK types) -- `src/packages/dumbo/src/core/schema/foreignKeys/index.ts` (new - barrel export) +- `src/packages/dumbo/src/core/schema/relationships/index.ts` (new - barrel export) **Deliverables:** -- Updated `dumboTable` signature to accept and capture `foreignKeys` generic +- Updated `dumboTable` signature to accept and capture `relationships` generic - Updated `dumboDatabase` signature with FK validation constraint - Updated `dumboDatabaseSchema` signature with FK validation constraint - Proper type exports @@ -256,11 +256,11 @@ All must pass before proceeding. Create two new files: -1. src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyTypes.ts +1. src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts - Add placeholder comment: `// Foreign key type definitions` -2. src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.type.spec.ts +2. src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts - Import Expect, Equal, ExpectError from '../../testing' - Add comment placeholders for test sections: - Schema Structure Extraction @@ -300,7 +300,7 @@ All must pass before proceeding. Following TDD approach, add type tests then implement ExtractSchemaNames utility. -In foreignKeyValidation.type.spec.ts, add under 'Schema Structure Extraction' comment section: +In relationshipValidation.type.spec.ts, add under 'Schema Structure Extraction' comment section: ```typescript // Schema Structure Extraction @@ -319,7 +319,7 @@ type _DB2 = DatabaseSchemaComponent<{ type _Test2 = Expect, "public" | "analytics">>; ``` -Then in foreignKeyTypes.ts, implement: +Then in relationshipTypes.ts, implement: ```typescript import type { DatabaseSchemaComponent, DatabaseSchemas } from "../components"; @@ -356,7 +356,7 @@ All must pass before proceeding. Following TDD approach, add type tests then implement ExtractTableNames utility. -In foreignKeyValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: +In relationshipValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: ```typescript void it("should extract table names from single-table schema", () => { @@ -380,7 +380,7 @@ void it("should extract table names from multi-table schema", () => { }); ``` -Then in foreignKeyTypes.ts, implement: +Then in relationshipTypes.ts, implement: ```typescript import type { @@ -422,7 +422,7 @@ All must pass before proceeding. Following TDD approach, add type tests then implement ExtractColumnNames utility. -In foreignKeyValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: +In relationshipValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: ```typescript void it("should extract column names from single-column table", () => { @@ -447,7 +447,7 @@ void it("should extract column names from multi-column table", () => { }); ``` -Then in foreignKeyTypes.ts, implement: +Then in relationshipTypes.ts, implement: ```typescript import type { @@ -488,7 +488,7 @@ All must pass before proceeding. Following TDD approach, add type test then implement AllColumnReferences for single schema. -In foreignKeyValidation.type.spec.ts, add to 'Column Reference Generation' describe block: +In relationshipValidation.type.spec.ts, add to 'Column Reference Generation' describe block: ```typescript import { SQL } from "../../../sql"; @@ -546,7 +546,7 @@ void it("should generate column references for single schema with multiple table }); ``` -Then in foreignKeyTypes.ts, implement: +Then in relationshipTypes.ts, implement: ```typescript export type AllColumnReferences = @@ -596,7 +596,7 @@ All must pass before proceeding. Add type test for multi-schema database to verify AllColumnReferences works correctly. -In foreignKeyValidation.type.spec.ts, add to 'Column Reference Generation' describe block: +In relationshipValidation.type.spec.ts, add to 'Column Reference Generation' describe block: ```typescript void it("should generate column references for multi-schema database", () => { @@ -653,29 +653,29 @@ All must pass before proceeding. --- -### Step 8: Define ForeignKeyDefinition Type +### Step 8: Define RelationshipDefinition Type **Context:** Define the structure for foreign key definitions that tables will use. **Prompt:** ``` -Add the ForeignKeyDefinition type to foreignKeyTypes.ts and create a basic type test. +Add the RelationshipDefinition type to relationshipTypes.ts and create a basic type test. -In foreignKeyTypes.ts, add: +In relationshipTypes.ts, add: ```typescript -export type ForeignKeyDefinition = { +export type RelationshipDefinition = { readonly columns: readonly string[]; readonly references: readonly string[]; }; ``` -In foreignKeyValidation.type.spec.ts, add to 'Foreign Key Definition Structure' describe block: +In relationshipValidation.type.spec.ts, add to 'Foreign Key Definition Structure' describe block: ```typescript void it("should accept valid foreign key definition", () => { - type FK = ForeignKeyDefinition; + type FK = RelationshipDefinition; const validFK: FK = { columns: ["user_id"], @@ -687,7 +687,7 @@ void it("should accept valid foreign key definition", () => { }); void it("should accept composite foreign key definition", () => { - type FK = ForeignKeyDefinition; + type FK = RelationshipDefinition; const compositeFK: FK = { columns: ["user_id", "tenant_id"], @@ -712,7 +712,7 @@ All must pass before proceeding. ``` **Acceptance Criteria:** -- ForeignKeyDefinition type added +- RelationshipDefinition type added - Basic structure tests pass - All quality gates pass @@ -725,29 +725,31 @@ All must pass before proceeding. **Prompt:** ``` -Update TableSchemaComponent to support an optional foreignKeys property. +Update TableSchemaComponent to support an optional relationships property. In src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts: -1. Import ForeignKeyDefinition: +1. Import RelationshipDefinition: ```typescript -import type { ForeignKeyDefinition } from "../foreignKeys/foreignKeyTypes"; +import type { RelationshipDefinition } from "../relationships/relationshipTypes"; ``` -2. Add generic parameter for foreign keys and foreignKeys property: +2. Add generic parameter for foreign keys and relationships property: ```typescript export type TableSchemaComponent< Columns extends TableColumns = TableColumns, - ForeignKeys extends readonly ForeignKeyDefinition[] = readonly ForeignKeyDefinition[] + Relationships extends readonly RelationshipDefinition[] = readonly RelationshipDefinition[] > = SchemaComponent< TableURN, Readonly<{ tableName: string; columns: ReadonlyMap & Columns; - primaryKey: TableColumnNames>[]; - foreignKeys?: ForeignKeys; + primaryKey: TableColumnNames< + TableSchemaComponent + >[]; + relationships?: Relationships; indexes: ReadonlyMap; addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; @@ -761,7 +763,7 @@ export type TableSchemaComponent< export type AnyTableSchemaComponent = TableSchemaComponent; ``` -4. Update tableSchemaComponent function signature and implementation to accept and return foreignKeys. +4. Update tableSchemaComponent function signature and implementation to accept and return relationships. After implementation, run quality gates as subagents: @@ -774,7 +776,7 @@ All must pass before proceeding. ``` **Acceptance Criteria:** -- TableSchemaComponent updated with foreignKeys support +- TableSchemaComponent updated with relationships support - Generic parameter added for type safety - No breaking changes to existing code - All quality gates pass @@ -788,17 +790,17 @@ All must pass before proceeding. **Prompt:** ``` -Update dumboTable function in dumboSchema.ts to accept foreignKeys parameter. +Update dumboTable function in dumboSchema.ts to accept relationships parameter. In src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts: -1. Import ForeignKeyDefinition at the top: +1. Import RelationshipDefinition at the top: ```typescript -import type { ForeignKeyDefinition } from "../foreignKeys/foreignKeyTypes"; +import type { RelationshipDefinition } from "../relationships/relationshipTypes"; ``` -2. Update dumboTable function to accept foreignKeys in definition parameter and pass it to tableSchemaComponent. +2. Update dumboTable function to accept relationships in definition parameter and pass it to tableSchemaComponent. After implementation, run quality gates as subagents: @@ -811,25 +813,25 @@ All must pass before proceeding. ``` **Acceptance Criteria:** -- dumboTable accepts foreignKeys parameter +- dumboTable accepts relationships parameter - Generic inference captures exact FK types - No breaking changes - All quality gates pass --- -### Step 11: Implement ValidateForeignKeyLength +### Step 11: Implement ValidateRelationshipLength **Context:** Validate that foreign key columns and references arrays have matching lengths. **Prompt:** ``` -Following TDD, add type tests then implement ValidateForeignKeyLength. +Following TDD, add type tests then implement ValidateRelationshipLength. -Create new file src/packages/dumbo/src/core/schema/foreignKeys/foreignKeyValidation.ts with validation result types and ValidateForeignKeyLength. +Create new file src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts with validation result types and ValidateRelationshipLength. -In foreignKeyValidation.type.spec.ts, add to 'Foreign Key Validation - Invalid Cases' describe block: +In relationshipValidation.type.spec.ts, add to 'Foreign Key Validation - Invalid Cases' describe block: ```typescript void it("should error when columns and references have different lengths", () => { @@ -838,7 +840,7 @@ void it("should error when columns and references have different lengths", () => references: ["public.users.id"]; }; - type Result = ValidateForeignKeyLength; + type Result = ValidateRelationshipLength; type Test = ExpectError; }); @@ -853,8 +855,8 @@ void it("should pass when columns and references have same length", () => { references: ["public.users.id", "public.users.tenant_id"]; }; - type Result1 = ValidateForeignKeyLength; - type Result2 = ValidateForeignKeyLength; + type Result1 = ValidateRelationshipLength; + type Result2 = ValidateRelationshipLength; type Test1 = Expect>; type Test2 = Expect>; }); @@ -872,23 +874,23 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateForeignKeyLength correctly validates array lengths +- ValidateRelationshipLength correctly validates array lengths - All quality gates pass --- -### Step 12: Implement ValidateForeignKeyColumns +### Step 12: Implement ValidateRelationshipColumns **Context:** Validate that all columns in a foreign key exist in the table. **Prompt:** ``` -Following TDD, add type tests then implement ValidateForeignKeyColumns. +Following TDD, add type tests then implement ValidateRelationshipColumns. -In foreignKeyValidation.type.spec.ts, add tests to both valid and invalid cases describe blocks. +In relationshipValidation.type.spec.ts, add tests to both valid and invalid cases describe blocks. -In foreignKeyValidation.ts, implement ValidateForeignKeyColumns with helper types to check if all elements of a tuple are in a union and to find invalid columns. +In relationshipValidation.ts, implement ValidateRelationshipColumns with helper types to check if all elements of a tuple are in a union and to find invalid columns. After implementation, run quality gates as subagents: @@ -902,24 +904,24 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateForeignKeyColumns correctly validates column existence +- ValidateRelationshipColumns correctly validates column existence - Helpful error messages show which columns are invalid - All quality gates pass --- -### Step 13: Implement ValidateForeignKeyReferences +### Step 13: Implement ValidateRelationshipReferences **Context:** Validate that all references in a foreign key point to valid schema.table.column paths. **Prompt:** ``` -Following TDD, add type tests then implement ValidateForeignKeyReferences. +Following TDD, add type tests then implement ValidateRelationshipReferences. -In foreignKeyValidation.type.spec.ts, add tests for valid and invalid reference scenarios. +In relationshipValidation.type.spec.ts, add tests for valid and invalid reference scenarios. -In foreignKeyValidation.ts, implement ValidateForeignKeyReferences with helper to find invalid references. +In relationshipValidation.ts, implement ValidateRelationshipReferences with helper to find invalid references. After implementation, run quality gates as subagents: @@ -933,24 +935,24 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateForeignKeyReferences correctly validates reference paths +- ValidateRelationshipReferences correctly validates reference paths - Helpful error messages show invalid references and available options - All quality gates pass --- -### Step 14: Implement ValidateSingleForeignKey (Combine Validations) +### Step 14: Implement ValidateSingleRelationship (Combine Validations) **Context:** Combine all FK validations into a single validation function. **Prompt:** ``` -Following TDD, add type tests then implement ValidateSingleForeignKey that combines all validations. +Following TDD, add type tests then implement ValidateSingleRelationship that combines all validations. -In foreignKeyValidation.type.spec.ts, add tests for complete FK validation covering valid cases and all error scenarios. +In relationshipValidation.type.spec.ts, add tests for complete FK validation covering valid cases and all error scenarios. -In foreignKeyValidation.ts, implement ValidateSingleForeignKey that chains length, columns, and references validation. +In relationshipValidation.ts, implement ValidateSingleRelationship that chains length, columns, and references validation. After implementation, run quality gates as subagents: @@ -964,24 +966,24 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateSingleForeignKey checks all validation rules in order +- ValidateSingleRelationship checks all validation rules in order - Returns first error encountered (fail fast) - All quality gates pass --- -### Step 15: Implement ValidateTableForeignKeys +### Step 15: Implement ValidateTableRelationships **Context:** Validate all foreign keys defined in a single table. **Prompt:** ``` -Following TDD, add type tests then implement ValidateTableForeignKeys. +Following TDD, add type tests then implement ValidateTableRelationships. -In foreignKeyValidation.type.spec.ts, add tests for tables with no FKs, single FK, and multiple FKs. +In relationshipValidation.type.spec.ts, add tests for tables with no FKs, single FK, and multiple FKs. -In foreignKeyValidation.ts, implement ValidateTableForeignKeys with helper to iterate through FK array. +In relationshipValidation.ts, implement ValidateTableRelationships with helper to iterate through FK array. After implementation, run quality gates as subagents: @@ -995,24 +997,24 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateTableForeignKeys handles tables with 0, 1, or multiple FKs +- ValidateTableRelationships handles tables with 0, 1, or multiple FKs - Returns first error encountered across all FKs - All quality gates pass --- -### Step 16: Implement ValidateSchemaForeignKeys +### Step 16: Implement ValidateSchemaRelationships **Context:** Validate all foreign keys across all tables in a schema. **Prompt:** ``` -Following TDD, add type tests then implement ValidateSchemaForeignKeys. +Following TDD, add type tests then implement ValidateSchemaRelationships. -In foreignKeyValidation.type.spec.ts, add tests for schemas with multiple tables and FKs. +In relationshipValidation.type.spec.ts, add tests for schemas with multiple tables and FKs. -In foreignKeyValidation.ts, implement ValidateSchemaForeignKeys with helper to iterate through tables. +In relationshipValidation.ts, implement ValidateSchemaRelationships with helper to iterate through tables. After implementation, run quality gates as subagents: @@ -1026,24 +1028,24 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateSchemaForeignKeys iterates through all tables +- ValidateSchemaRelationships iterates through all tables - Returns first error encountered - All quality gates pass --- -### Step 17: Implement ValidateDatabaseForeignKeys +### Step 17: Implement ValidateDatabaseRelationships **Context:** Validate all foreign keys across all schemas in the database - the top-level validation. **Prompt:** ``` -Following TDD, add type tests then implement ValidateDatabaseForeignKeys. +Following TDD, add type tests then implement ValidateDatabaseRelationships. -In foreignKeyValidation.type.spec.ts, add to 'Integration Tests' describe block tests for complete database validation including self-referential FKs. +In relationshipValidation.type.spec.ts, add to 'Integration Tests' describe block tests for complete database validation including self-referential FKs. -In foreignKeyValidation.ts, implement ValidateDatabaseForeignKeys that validates all schemas using AllColumnReferences. +In relationshipValidation.ts, implement ValidateDatabaseRelationships that validates all schemas using AllColumnReferences. After implementation, run quality gates as subagents: @@ -1057,7 +1059,7 @@ All must pass before proceeding. **Acceptance Criteria:** - Type tests added and pass -- ValidateDatabaseForeignKeys validates entire database +- ValidateDatabaseRelationships validates entire database - Handles self-referential FKs correctly - Returns first error encountered - All quality gates pass @@ -1071,7 +1073,7 @@ All must pass before proceeding. **Prompt:** ``` -Create src/packages/dumbo/src/core/schema/foreignKeys/index.ts to export all foreign key types. +Create src/packages/dumbo/src/core/schema/relationships/index.ts to export all foreign key types. Update src/packages/dumbo/src/core/schema/components/index.ts to re-export FK types. @@ -1083,7 +1085,7 @@ After implementation, run quality gates as subagents: All must pass before proceeding. -``` +```` **Acceptance Criteria:** - Barrel export file created @@ -1103,12 +1105,13 @@ Use conditional return types on `database()` function overloads: ```typescript function dumboDatabase(schemas: Schemas): - ValidateDatabaseForeignKeys> extends { valid: true } + ValidateDatabaseRelationships> extends { valid: true } ? DatabaseSchemaComponent - : ValidateDatabaseForeignKeys>; -``` + : ValidateDatabaseRelationships>; +```` **How it works:** + 1. User calls `const db = database({...})` 2. TypeScript infers exact `Schemas` type from literal 3. Return type is evaluated: @@ -1121,13 +1124,14 @@ function dumboDatabase(schemas: Schemas): **No `as const` needed** - Generic inference on `table()` already captures exact tuple types from array literals. **Prompt:** -``` + +```` IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. ## Part 1: Add Failing Type Tests -Add to foreignKeyValidation.type.spec.ts: +Add to relationshipValidation.type.spec.ts: ```typescript // TEST: Invalid column should cause type error at database() call @@ -1138,7 +1142,7 @@ const _dbInvalidColumn = database('test', { id: column('id', Varchar('max')), user_id: column('id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['invalid_col'], references: ['public.users.id'] }, ], }), @@ -1161,7 +1165,7 @@ const _dbValid = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + relationships: [{ columns: ['user_id'], references: ['public.users.id'] }], }), }), }); @@ -1170,7 +1174,7 @@ type _ValidResult = typeof _dbValid; type _Test_Valid = Expect< Equal<_ValidResult, DatabaseSchemaComponent> >; // This should PASS -``` +```` Run `npm run build:ts` - you should see errors because validation not wired yet. @@ -1178,21 +1182,24 @@ Run `npm run build:ts` - you should see errors because validation not wired yet. Update dumboDatabase function in dumboSchema.ts to add FK validation via conditional return types. -1. Import ValidateDatabaseForeignKeys from '../foreignKeys' +1. Import ValidateDatabaseRelationships from '../relationships' 2. Find the TWO overload signatures that accept `schemas: Schemas` parameter (around lines 137 and 147) + - Do NOT modify overloads that accept single `schema` parameter 3. Change their return type from: + ```typescript - DatabaseSchemaComponent + DatabaseSchemaComponent; ``` To: + ```typescript - ValidateDatabaseForeignKeys> extends { valid: true } + ValidateDatabaseRelationships> extends { valid: true } ? DatabaseSchemaComponent - : ValidateDatabaseForeignKeys> + : ValidateDatabaseRelationships> ``` 4. Update the implementation function's return statement (around line 197) to cast the result: @@ -1201,11 +1208,13 @@ Update dumboDatabase function in dumboSchema.ts to add FK validation via conditi databaseName, schemas: schemaMap as Schemas, ...dbOptions, - }) as ValidateDatabaseForeignKeys< + }) as ValidateDatabaseRelationships< DatabaseSchemaComponent - > extends { valid: true } + > extends { + valid: true; + } ? DatabaseSchemaComponent - : ValidateDatabaseForeignKeys>; + : ValidateDatabaseRelationships>; ``` After implementation, run quality gates as subagents: @@ -1245,18 +1254,18 @@ IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. ## Part 1: Add Failing Type Tests -Add to foreignKeyValidation.type.spec.ts: +Add to relationshipValidation.type.spec.ts: ```typescript // TEST: Invalid intra-schema FK at schema() level -const _schemaInvalidFK = schema('public', { - posts: table('posts', { +const _schemaInvalidFK = schema("public", { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['invalid_col'], references: ['public.users.id'] }, + relationships: [ + { columns: ["invalid_col"], references: ["public.users.id"] }, ], }), }); @@ -1267,16 +1276,16 @@ type _Test_SchemaInvalid = Expect< >; // Should FAIL // TEST: Valid intra-schema FK -const _schemaValidFK = schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, +const _schemaValidFK = schema("public", { + users: table("users", { + columns: { id: column("id", Varchar("max")) }, }), - posts: table('posts', { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + relationships: [{ columns: ["user_id"], references: ["public.users.id"] }], }), }); @@ -1292,23 +1301,25 @@ Run `npm run build:ts` - should see errors because schema validation not wired y Update dumboDatabaseSchema function in dumboSchema.ts to add FK validation for intra-schema references. -1. Import ValidateSchemaForeignKeys from '../foreignKeys' +1. Import ValidateSchemaRelationships from '../relationships' 2. Find the TWO overload signatures that accept `tables: Tables` parameter 3. Change their return type from: + ```typescript - DatabaseSchemaSchemaComponent + DatabaseSchemaSchemaComponent; ``` To: + ```typescript - ValidateSchemaForeignKeys< + ValidateSchemaRelationships< DatabaseSchemaSchemaComponent, AllColumnReferences> > extends { valid: true } ? DatabaseSchemaSchemaComponent - : ValidateSchemaForeignKeys< + : ValidateSchemaRelationships< DatabaseSchemaSchemaComponent, AllColumnReferences> > @@ -1381,7 +1392,7 @@ All must pass before proceeding. **Prompt:** ``` -Add comprehensive type-level validation tests to foreignKeyValidation.type.spec.ts (after the basic type utility tests). +Add comprehensive type-level validation tests to relationshipValidation.type.spec.ts (after the basic type utility tests). Add these test cases demonstrating compile-time FK validation: @@ -1390,19 +1401,19 @@ Add these test cases demonstrating compile-time FK validation: // COMPILE-TIME FK VALIDATION TESTS // ============================================================================ -import { dumboSchema } from '../dumboSchema/dumboSchema'; +import { dumboSchema } from "../dumboSchema/dumboSchema"; const { database, schema, table, column } = dumboSchema; // TEST 1: Invalid column name - should be error type -const _dbInvalidColumn = database('test', { - public: schema('public', { - posts: table('posts', { +const _dbInvalidColumn = database("test", { + public: schema("public", { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['nonexistent_col'], references: ['public.users.id'] }, + relationships: [ + { columns: ["nonexistent_col"], references: ["public.users.id"] }, ], }), }), @@ -1417,18 +1428,18 @@ type _Test_InvalidColumn = Expect< : true; // Should be true (types don't match) // TEST 2: Invalid reference path - should be error type -const _dbInvalidRef = database('test', { - public: schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, +const _dbInvalidRef = database("test", { + public: schema("public", { + users: table("users", { + columns: { id: column("id", Varchar("max")) }, }), - posts: table('posts', { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['user_id'], references: ['public.nonexistent.id'] }, + relationships: [ + { columns: ["user_id"], references: ["public.nonexistent.id"] }, ], }), }), @@ -1442,22 +1453,22 @@ type _Test_InvalidRef = Expect< : true; // Should be true // TEST 3: Length mismatch - should be error type -const _dbLengthMismatch = database('test', { - public: schema('public', { - users: table('users', { +const _dbLengthMismatch = database("test", { + public: schema("public", { + users: table("users", { columns: { - id: column('id', Varchar('max')), - tenant_id: column('tenant_id', Varchar('max')), + id: column("id", Varchar("max")), + tenant_id: column("tenant_id", Varchar("max")), }, }), - posts: table('posts', { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - tenant_id: column('tenant_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), + tenant_id: column("tenant_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['user_id', 'tenant_id'], references: ['public.users.id'] }, + relationships: [ + { columns: ["user_id", "tenant_id"], references: ["public.users.id"] }, ], }), }), @@ -1471,45 +1482,45 @@ type _Test_LengthMismatch = Expect< : true; // Should be true // TEST 4: Valid FK - should work perfectly (type matches) -const _dbValidFK = database('test', { - public: schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, +const _dbValidFK = database("test", { + public: schema("public", { + users: table("users", { + columns: { id: column("id", Varchar("max")) }, }), - posts: table('posts', { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + relationships: [ + { columns: ["user_id"], references: ["public.users.id"] }, + ], }), }), }); type _ValidFKType = typeof _dbValidFK; -type _Test_ValidFK = Expect< - Equal<_ValidFKType, DatabaseSchemaComponent> ->; // Should PASS (types match) +type _Test_ValidFK = Expect>>; // Should PASS (types match) // TEST 5: Composite FK - should work -const _dbCompositeFK = database('test', { - public: schema('public', { - users: table('users', { +const _dbCompositeFK = database("test", { + public: schema("public", { + users: table("users", { columns: { - id: column('id', Varchar('max')), - tenant_id: column('tenant_id', Varchar('max')), + id: column("id", Varchar("max")), + tenant_id: column("tenant_id", Varchar("max")), }, }), - posts: table('posts', { + posts: table("posts", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - tenant_id: column('tenant_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), + tenant_id: column("tenant_id", Varchar("max")), }, - foreignKeys: [ + relationships: [ { - columns: ['user_id', 'tenant_id'], - references: ['public.users.id', 'public.users.tenant_id'], + columns: ["user_id", "tenant_id"], + references: ["public.users.id", "public.users.tenant_id"], }, ], }), @@ -1522,40 +1533,38 @@ type _Test_CompositeFK = Expect< >; // Should PASS // TEST 6: Self-referential FK - should work -const _dbSelfRef = database('test', { - public: schema('public', { - users: table('users', { +const _dbSelfRef = database("test", { + public: schema("public", { + users: table("users", { columns: { - id: column('id', Varchar('max')), - manager_id: column('manager_id', Varchar('max')), + id: column("id", Varchar("max")), + manager_id: column("manager_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['manager_id'], references: ['public.users.id'] }, + relationships: [ + { columns: ["manager_id"], references: ["public.users.id"] }, ], }), }), }); type _SelfRefType = typeof _dbSelfRef; -type _Test_SelfRef = Expect< - Equal<_SelfRefType, DatabaseSchemaComponent> ->; // Should PASS +type _Test_SelfRef = Expect>>; // Should PASS // TEST 7: Cross-schema FK - should work -const _dbCrossSchema = database('test', { - public: schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, +const _dbCrossSchema = database("test", { + public: schema("public", { + users: table("users", { + columns: { id: column("id", Varchar("max")) }, }), }), - analytics: schema('analytics', { - events: table('events', { + analytics: schema("analytics", { + events: table("events", { columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), + id: column("id", Varchar("max")), + user_id: column("user_id", Varchar("max")), }, - foreignKeys: [ - { columns: ['user_id'], references: ['public.users.id'] }, + relationships: [ + { columns: ["user_id"], references: ["public.users.id"] }, ], }), }), diff --git a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md index 9299d4c5..33e3b756 100644 --- a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md +++ b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md @@ -1098,7 +1098,7 @@ export class MigrationOrchestrator { await this.runMigrationGroup(dumbo, grouped.tables); await this.runMigrationGroup(dumbo, grouped.columns); await this.runMigrationGroup(dumbo, grouped.indexes); - // Future: grouped.foreignKeys, grouped.constraints + // Future: grouped.relationships, grouped.constraints // Update snapshot after successful migration await this.updateSnapshot(); diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts b/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts deleted file mode 100644 index c99d8180..00000000 --- a/src/packages/dumbo/src/core/schema/components/foreignKeys/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './foreignKeyTypes'; - -export * from './foreignKeyValidation'; diff --git a/src/packages/dumbo/src/core/schema/components/index.ts b/src/packages/dumbo/src/core/schema/components/index.ts index 1ae8b8cb..f1bc6b9d 100644 --- a/src/packages/dumbo/src/core/schema/components/index.ts +++ b/src/packages/dumbo/src/core/schema/components/index.ts @@ -7,8 +7,8 @@ import { TableURN } from './tableSchemaComponent'; export * from './columnSchemaComponent'; export * from './databaseSchemaComponent'; export * from './databaseSchemaSchemaComponent'; -export * from './foreignKeys'; export * from './indexSchemaComponent'; +export * from './relationships'; export * from './tableSchemaComponent'; export * from './tableTypesInference'; diff --git a/src/packages/dumbo/src/core/schema/components/relationships/index.ts b/src/packages/dumbo/src/core/schema/components/relationships/index.ts new file mode 100644 index 00000000..e29b349e --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/index.ts @@ -0,0 +1,3 @@ +export * from './relationshipTypes'; + +export * from './relationshipValidation'; diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts similarity index 90% rename from src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts rename to src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts index efcc11a2..d6e5fe3c 100644 --- a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyTypes.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts @@ -8,7 +8,7 @@ import type { TableColumnNames, TableColumns, TableSchemaComponent, -} from '../'; +} from '..'; export type ExtractSchemaNames = DB extends DatabaseSchemaComponent @@ -47,12 +47,12 @@ export type AllColumnReferences = }[keyof Schemas] : never; -export type ForeignKeyDefinition = { +export type RelationshipDefinition = { readonly columns: readonly Columns[]; readonly references: readonly References[]; }; -export const foreignKey = < +export const relationship = < const Columns extends readonly string[], const References extends readonly string[], >( @@ -65,4 +65,4 @@ export const foreignKey = < } as const; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyForeignKeyDefinition = ForeignKeyDefinition; +export type AnyRelationshipDefinition = RelationshipDefinition; diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts similarity index 85% rename from src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts rename to src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts index 2d148abf..c3b16bf5 100644 --- a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts @@ -3,7 +3,7 @@ import type { AnyDatabaseSchemaSchemaComponent, DatabaseSchemaComponent, DatabaseSchemaSchemaComponent, -} from '../'; +} from '..'; import type { AnyTableSchemaComponent, TableSchemaComponent, @@ -11,8 +11,8 @@ import type { import type { TableColumnNames } from '../tableTypesInference'; import type { AllColumnReferences, - AnyForeignKeyDefinition, -} from './foreignKeyTypes'; + AnyRelationshipDefinition, +} from './relationshipTypes'; export type ValidationResult< Valid extends boolean, @@ -21,7 +21,7 @@ export type ValidationResult< type GetArrayLength = T['length']; -export type ValidateForeignKeyLength< +export type ValidateRelationshipLength< FK extends { columns: readonly unknown[]; references: readonly unknown[] }, > = GetArrayLength extends GetArrayLength @@ -56,7 +56,7 @@ type AllInTuple< : false : true; -export type ValidateForeignKeyColumns< +export type ValidateRelationshipColumns< FK extends { columns: readonly string[] }, ValidColumns extends string, > = @@ -81,7 +81,7 @@ type FindInvalidReferences< : Invalid : Invalid; -export type ValidateForeignKeyReferences< +export type ValidateRelationshipReferences< FK extends { references: readonly string[] }, ValidReferences extends string, > = @@ -92,32 +92,32 @@ export type ValidateForeignKeyReferences< `Invalid foreign key references: ${FindInvalidReferences extends infer Invalid ? (Invalid extends string[] ? Invalid[number] : never) : never}. Available references: ${ValidReferences}` >; -export type ValidateSingleForeignKey< +export type ValidateSingleRelationship< FK extends { columns: readonly string[]; references: readonly string[] }, TableColumns extends string, ValidReferences extends string, > = - ValidateForeignKeyLength extends { valid: false; error: infer E } + ValidateRelationshipLength extends { valid: false; error: infer E } ? ValidationResult - : ValidateForeignKeyColumns extends { + : ValidateRelationshipColumns extends { valid: false; error: infer E; } ? ValidationResult - : ValidateForeignKeyReferences extends { + : ValidateRelationshipReferences extends { valid: false; error: infer E; } ? ValidationResult : ValidationResult; -export type ValidateForeignKeyArray< - FKs extends readonly AnyForeignKeyDefinition[], +export type ValidateRelationshipArray< + FKs extends readonly AnyRelationshipDefinition[], TableColumns extends string, ValidReferences extends string, > = FKs extends readonly [] ? ValidationResult - : ValidateSingleForeignKey< + : ValidateSingleRelationship< FKs[number], TableColumns, ValidReferences @@ -128,12 +128,12 @@ export type ValidateForeignKeyArray< ? ValidationResult : ValidationResult; -export type ValidateTableForeignKeys< +export type ValidateTableRelationships< Table extends AnyTableSchemaComponent, ValidReferences extends string, > = Table extends TableSchemaComponent - ? ValidateForeignKeyArray< + ? ValidateRelationshipArray< FKs, TableColumnNames
& string, ValidReferences @@ -144,7 +144,7 @@ export type ValidateTablesInSchema< Tables extends Record, ValidReferences extends string, > = { - [TableName in keyof Tables]: ValidateTableForeignKeys< + [TableName in keyof Tables]: ValidateTableRelationships< Tables[TableName], ValidReferences >; @@ -156,7 +156,7 @@ export type ValidateTablesInSchema< : ValidationResult : ValidationResult; -export type ValidateSchemaForeignKeys< +export type ValidateSchemaRelationships< Schema extends AnyDatabaseSchemaSchemaComponent, ValidReferences extends string, > = @@ -168,7 +168,7 @@ export type ValidateSchemasInDatabase< Schemas extends Record, ValidReferences extends string, > = { - [SchemaName in keyof Schemas]: ValidateSchemaForeignKeys< + [SchemaName in keyof Schemas]: ValidateSchemaRelationships< Schemas[SchemaName], ValidReferences >; @@ -180,7 +180,9 @@ export type ValidateSchemasInDatabase< : ValidationResult : ValidationResult; -export type ValidateDatabaseForeignKeys = +export type ValidateDatabaseRelationships< + DB extends AnyDatabaseSchemaComponent, +> = DB extends DatabaseSchemaComponent ? ValidateSchemasInDatabase> : ValidationResult; diff --git a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts similarity index 82% rename from src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts rename to src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts index 4ec3283a..b8799cec 100644 --- a/src/packages/dumbo/src/core/schema/components/foreignKeys/foreignKeyValidation.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts @@ -6,7 +6,7 @@ import type { DatabaseSchemaComponent, DatabaseSchemaSchemaComponent, TableSchemaComponent, -} from '../'; +} from '..'; import { SQL } from '../../../sql'; import type { Equal, Expect } from '../../../testing'; import { dumboSchema } from '../../dumboSchema'; @@ -15,8 +15,8 @@ import type { ExtractColumnNames, ExtractSchemaNames, ExtractTableNames, - ForeignKeyDefinition, -} from './foreignKeyTypes'; + RelationshipDefinition, +} from './relationshipTypes'; const { database, schema, table, column } = dumboSchema; const { Varchar } = SQL.column.type; @@ -137,7 +137,7 @@ type _Test9 = Expect< > >; -const _validFK: ForeignKeyDefinition = { +const _validFK: RelationshipDefinition = { columns: ['user_id'], references: ['public.users.id'], }; @@ -145,7 +145,7 @@ const _validFK: ForeignKeyDefinition = { type _ColumnsType = typeof _validFK.columns; type _Test10 = Expect>; -const _compositeFK: ForeignKeyDefinition = { +const _compositeFK: RelationshipDefinition = { columns: ['user_id', 'tenant_id'], references: ['public.users.id', 'public.users.tenant_id'], }; @@ -157,16 +157,16 @@ type _Test12 = Expect>; import type { IsError } from '../../../testing/typesTesting'; import type { - ValidateDatabaseForeignKeys, - ValidateForeignKeyLength, -} from './foreignKeyValidation'; + ValidateDatabaseRelationships, + ValidateRelationshipLength, +} from './relationshipValidation'; type _FK_LengthMismatch = { columns: ['user_id', 'tenant_id']; references: ['public.users.id']; }; -type _Result_LengthMismatch = ValidateForeignKeyLength<_FK_LengthMismatch>; +type _Result_LengthMismatch = ValidateRelationshipLength<_FK_LengthMismatch>; type _Test13 = Expect>; type _FK_SingleMatch = { @@ -179,19 +179,19 @@ type _FK_CompositeMatch = { references: ['public.users.id', 'public.users.tenant_id']; }; -type _Result_SingleMatch = ValidateForeignKeyLength<_FK_SingleMatch>; -type _Result_CompositeMatch = ValidateForeignKeyLength<_FK_CompositeMatch>; +type _Result_SingleMatch = ValidateRelationshipLength<_FK_SingleMatch>; +type _Result_CompositeMatch = ValidateRelationshipLength<_FK_CompositeMatch>; type _Test14 = Expect>; type _Test15 = Expect>; -import type { ValidateForeignKeyColumns } from './foreignKeyValidation'; +import type { ValidateRelationshipColumns } from './relationshipValidation'; type _FK_InvalidColumn = { columns: ['user_id', 'invalid_col']; references: ['public.users.id', 'public.users.tenant_id']; }; -type _Result_InvalidColumn = ValidateForeignKeyColumns< +type _Result_InvalidColumn = ValidateRelationshipColumns< _FK_InvalidColumn, 'id' | 'email' | 'user_id' >; @@ -207,25 +207,25 @@ type _FK_ValidCompositeColumns = { references: ['public.users.id', 'public.users.email']; }; -type _Result_ValidColumns = ValidateForeignKeyColumns< +type _Result_ValidColumns = ValidateRelationshipColumns< _FK_ValidColumns, 'id' | 'email' | 'user_id' >; -type _Result_ValidCompositeColumns = ValidateForeignKeyColumns< +type _Result_ValidCompositeColumns = ValidateRelationshipColumns< _FK_ValidCompositeColumns, 'id' | 'email' | 'user_id' >; type _Test17 = Expect>; type _Test18 = Expect>; -import type { ValidateForeignKeyReferences } from './foreignKeyValidation'; +import type { ValidateRelationshipReferences } from './relationshipValidation'; type _FK_InvalidReference = { columns: ['user_id']; references: ['public.nonexistent.id']; }; -type _Result_InvalidReference = ValidateForeignKeyReferences< +type _Result_InvalidReference = ValidateRelationshipReferences< _FK_InvalidReference, 'public.users.id' | 'public.users.email' | 'public.posts.id' >; @@ -241,25 +241,25 @@ type _FK_ValidCompositeReference = { references: ['public.users.id', 'public.posts.id']; }; -type _Result_ValidReference = ValidateForeignKeyReferences< +type _Result_ValidReference = ValidateRelationshipReferences< _FK_ValidReference, 'public.users.id' | 'public.users.email' | 'public.posts.id' >; -type _Result_ValidCompositeReference = ValidateForeignKeyReferences< +type _Result_ValidCompositeReference = ValidateRelationshipReferences< _FK_ValidCompositeReference, 'public.users.id' | 'public.users.email' | 'public.posts.id' >; type _Test20 = Expect>; type _Test21 = Expect>; -import type { ValidateSingleForeignKey } from './foreignKeyValidation'; +import type { ValidateSingleRelationship } from './relationshipValidation'; type _FK_Complete_Valid = { columns: ['user_id']; references: ['public.users.id']; }; -type _Result_Complete_Valid = ValidateSingleForeignKey< +type _Result_Complete_Valid = ValidateSingleRelationship< _FK_Complete_Valid, 'id' | 'user_id', 'public.users.id' | 'public.users.email' @@ -271,7 +271,7 @@ type _FK_Complete_LengthError = { references: ['public.users.id']; }; -type _Result_Complete_LengthError = ValidateSingleForeignKey< +type _Result_Complete_LengthError = ValidateSingleRelationship< _FK_Complete_LengthError, 'id' | 'user_id' | 'tenant_id', 'public.users.id' | 'public.users.email' @@ -283,7 +283,7 @@ type _FK_Complete_ColumnError = { references: ['public.users.id']; }; -type _Result_Complete_ColumnError = ValidateSingleForeignKey< +type _Result_Complete_ColumnError = ValidateSingleRelationship< _FK_Complete_ColumnError, 'id' | 'user_id', 'public.users.id' | 'public.users.email' @@ -295,7 +295,7 @@ type _FK_Complete_ReferenceError = { references: ['public.invalid.id']; }; -type _Result_Complete_ReferenceError = ValidateSingleForeignKey< +type _Result_Complete_ReferenceError = ValidateSingleRelationship< _FK_Complete_ReferenceError, 'id' | 'user_id', 'public.users.id' | 'public.users.email' @@ -312,7 +312,7 @@ type _Test24A = Expect< > >; -import type { ValidateForeignKeyArray } from './foreignKeyValidation'; +import type { ValidateRelationshipArray } from './relationshipValidation'; type _FKArray_Mixed = readonly [ { @@ -325,7 +325,7 @@ type _FKArray_Mixed = readonly [ }, ]; -type _Result_FKArray_Mixed = ValidateForeignKeyArray< +type _Result_FKArray_Mixed = ValidateRelationshipArray< _FKArray_Mixed, 'id' | 'user_id', 'public.users.id' | 'public.users.email' @@ -355,28 +355,28 @@ type _FKArray_AllValid = readonly [ }, ]; -type _Result_FKArray_AllValid = ValidateForeignKeyArray< +type _Result_FKArray_AllValid = ValidateRelationshipArray< _FKArray_AllValid, 'id' | 'user_id' | 'email', 'public.users.id' | 'public.users.email' >; -type _ValidateForeignKeyArrayResult_InvalidFK = ValidateForeignKeyArray< +type _ValidateRelationshipArrayResult_InvalidFK = ValidateRelationshipArray< [{ columns: ['invalid']; references: ['public.users.id'] }], 'id' | 'user_id', 'public.users.id' | 'public.users.email' >; -type _TestValidateForeignKeyArrayResult_InvalidFK = Expect< - IsError<_ValidateForeignKeyArrayResult_InvalidFK> +type _TestValidateRelationshipArrayResult_InvalidFK = Expect< + IsError<_ValidateRelationshipArrayResult_InvalidFK> >; -import type { ValidateTableForeignKeys } from './foreignKeyValidation'; +import type { ValidateTableRelationships } from './relationshipValidation'; type _Table_NoFKs = TableSchemaComponent<{ id: AnyColumnSchemaComponent; email: AnyColumnSchemaComponent; }>; -type _Result_NoFKs = ValidateTableForeignKeys< +type _Result_NoFKs = ValidateTableRelationships< _Table_NoFKs, 'public.users.id' | 'public.users.email' >; @@ -390,7 +390,7 @@ type _Table_SingleFK = TableSchemaComponent< [{ columns: ['user_id']; references: ['public.users.id'] }] >; -type _Result_SingleFK = ValidateTableForeignKeys< +type _Result_SingleFK = ValidateTableRelationships< _Table_SingleFK, 'public.users.id' | 'public.users.email' >; @@ -408,7 +408,7 @@ type _Table_MultipleFK = TableSchemaComponent< ] >; -type _Result_MultipleFK = ValidateTableForeignKeys< +type _Result_MultipleFK = ValidateTableRelationships< _Table_MultipleFK, 'public.users.id' | 'public.users.email' >; @@ -422,13 +422,13 @@ type _Table_InvalidFK = TableSchemaComponent< [{ readonly columns: ['id']; references: readonly ['public.users.id'] }] >; -type _Result_InvalidFK = ValidateTableForeignKeys< +type _Result_InvalidFK = ValidateTableRelationships< _Table_InvalidFK, 'public.posts.id' | 'public.users.email' >; type _Test29 = Expect>; -import type { ValidateSchemaForeignKeys } from './foreignKeyValidation'; +import type { ValidateSchemaRelationships } from './relationshipValidation'; type _Schema_MultiTable = DatabaseSchemaSchemaComponent<{ users: TableSchemaComponent<{ @@ -444,7 +444,7 @@ type _Schema_MultiTable = DatabaseSchemaSchemaComponent<{ >; }>; -type _Result_Schema_Valid = ValidateSchemaForeignKeys< +type _Result_Schema_Valid = ValidateSchemaRelationships< _Schema_MultiTable, 'public.users.id' | 'public.users.email' | 'public.posts.id' >; @@ -460,7 +460,7 @@ type _Schema_WithError = DatabaseSchemaSchemaComponent<{ >; }>; -type _Result_Schema_Error = ValidateSchemaForeignKeys< +type _Result_Schema_Error = ValidateSchemaRelationships< _Schema_WithError, 'public.posts.id' | 'public.users.email' >; @@ -473,7 +473,7 @@ const _dbWithErrorVSInDB = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: [{ columns: ['id'], references: ['public.users.id'] }], }), }), }); @@ -495,12 +495,14 @@ const _fullDb = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + relationships: [ + { columns: ['user_id'], references: ['public.users.id'] }, + ], }), }), }); -type _Result_FullDb = ValidateDatabaseForeignKeys; +type _Result_FullDb = ValidateDatabaseRelationships; type _Test32 = Expect>; const _dbWithSelfRef = database('test', { @@ -510,14 +512,14 @@ const _dbWithSelfRef = database('test', { id: column('id', Varchar('max')), manager_id: column('manager_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['manager_id'], references: ['public.users.id'] }, ], }), }), }); -type _Result_SelfRef = ValidateDatabaseForeignKeys; +type _Result_SelfRef = ValidateDatabaseRelationships; type _Test33 = Expect>; const _dbWithError = database('test', { @@ -527,7 +529,7 @@ const _dbWithError = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: [{ columns: ['id'], references: ['public.users.id'] }], }), }), }); @@ -542,7 +544,7 @@ const _dbInvalidColumn = database('test', { id: column('id', Varchar('max')), user_id: column('id', Varchar('max')), }, - foreignKeys: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: [{ columns: ['id'], references: ['public.users.id'] }], }), }), }); @@ -561,7 +563,9 @@ const _dbValid = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [{ columns: ['user_id'], references: ['public.users.id'] }], + relationships: [ + { columns: ['user_id'], references: ['public.users.id'] }, + ], }), }), }); diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index eba4d7fb..8b0eb53c 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -8,11 +8,11 @@ import { ColumnURNType, type AnyColumnSchemaComponent, } from './columnSchemaComponent'; -import type { ForeignKeyDefinition } from './foreignKeys/foreignKeyTypes'; import { IndexURNType, type IndexSchemaComponent, } from './indexSchemaComponent'; +import type { RelationshipDefinition } from './relationships/relationshipTypes'; import type { TableColumnNames } from './tableTypesInference'; export type TableURNType = 'sc:dumbo:table'; @@ -23,31 +23,33 @@ export const TableURN = ({ name }: { name: string }): TableURN => `${TableURNType}:${name}`; export type TableColumns = Record; -export type TableForeignKeys< +export type TableRelationships< Columns extends string = string, // eslint-disable-next-line @typescript-eslint/no-explicit-any - FKS extends ForeignKeyDefinition = ForeignKeyDefinition< + FKS extends RelationshipDefinition = RelationshipDefinition< Columns, // eslint-disable-next-line @typescript-eslint/no-explicit-any any >, > = - FKS extends ForeignKeyDefinition - ? readonly ForeignKeyDefinition[] + FKS extends RelationshipDefinition + ? readonly RelationshipDefinition[] : never; export type TableSchemaComponent< Columns extends TableColumns = TableColumns, - ForeignKeys extends TableForeignKeys< + Relationships extends TableRelationships< keyof Columns & string - > = TableForeignKeys, + > = TableRelationships, > = SchemaComponent< TableURN, Readonly<{ tableName: string; columns: ReadonlyMap & Columns; - primaryKey: TableColumnNames>[]; - foreignKeys: ForeignKeys; + primaryKey: TableColumnNames< + TableSchemaComponent + >[]; + relationships: Relationships; indexes: ReadonlyMap; addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; @@ -59,23 +61,23 @@ export type AnyTableSchemaComponent = TableSchemaComponent; export const tableSchemaComponent = < Columns extends TableColumns = TableColumns, - const ForeignKeys extends TableForeignKeys = TableForeignKeys, + const Relationships extends TableRelationships = TableRelationships, >({ tableName, columns, primaryKey, - foreignKeys, + relationships, ...migrationsOrComponents }: { tableName: string; columns?: Columns; - primaryKey?: TableColumnNames>[]; - foreignKeys?: ForeignKeys; -} & SchemaComponentOptions): TableSchemaComponent & { - foreignKeys: ForeignKeys; + primaryKey?: TableColumnNames>[]; + relationships?: Relationships; +} & SchemaComponentOptions): TableSchemaComponent & { + relationships: Relationships; } => { columns ??= {} as Columns; - foreignKeys ??= {} as ForeignKeys; + relationships ??= {} as Relationships; const base = schemaComponent(TableURN({ name: tableName }), { migrations: migrationsOrComponents.migrations ?? [], @@ -89,7 +91,7 @@ export const tableSchemaComponent = < ...base, tableName, primaryKey: primaryKey ?? [], - foreignKeys, + relationships, get columns() { const columnsMap = mapSchemaComponentsOfType( base.components, @@ -108,7 +110,7 @@ export const tableSchemaComponent = < }, addColumn: (column: AnyColumnSchemaComponent) => base.addComponent(column), addIndex: (index: IndexSchemaComponent) => base.addComponent(index), - } as TableSchemaComponent & { - foreignKeys: ForeignKeys; + } as TableSchemaComponent & { + relationships: Relationships; }; }; diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index 3c6be831..d67b99eb 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,5 +1,5 @@ import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; -import type { ValidateDatabaseForeignKeys } from '../components'; +import type { ValidateDatabaseRelationships } from '../components'; import { type AnyDatabaseSchemaSchemaComponent, columnSchemaComponent, @@ -14,7 +14,7 @@ import { type IndexSchemaComponent, type TableColumnNames, type TableColumns, - type TableForeignKeys, + type TableRelationships, tableSchemaComponent, type TableSchemaComponent, } from '../components'; @@ -61,19 +61,22 @@ const dumboIndex = ( const dumboTable = < Columns extends TableColumns = TableColumns, - const ForeignKeys extends TableForeignKeys< + const Relationships extends TableRelationships< keyof Columns & string - > = TableForeignKeys, + > = TableRelationships, >( name: string, definition: { columns?: Columns; - primaryKey?: TableColumnNames>[]; - foreignKeys?: ForeignKeys; + primaryKey?: TableColumnNames< + TableSchemaComponent + >[]; + relationships?: Relationships; indexes?: Record; } & SchemaComponentOptions, -): TableSchemaComponent => { - const { columns, indexes, primaryKey, foreignKeys, ...options } = definition; +): TableSchemaComponent => { + const { columns, indexes, primaryKey, relationships, ...options } = + definition; const components = [...(indexes ? Object.values(indexes) : [])]; @@ -81,7 +84,7 @@ const dumboTable = < tableName: name, columns: columns ?? ({} as Columns), primaryKey: primaryKey ?? [], - ...(foreignKeys !== undefined ? { foreignKeys } : {}), + ...(relationships !== undefined ? { relationships } : {}), components, ...options, }); @@ -138,11 +141,11 @@ dumboDatabaseSchema.from = ( type ValidatedDatabaseSchemaComponent< Schemas extends DatabaseSchemas = DatabaseSchemas, > = - ValidateDatabaseForeignKeys> extends { + ValidateDatabaseRelationships> extends { valid: true; } ? DatabaseSchemaComponent - : ValidateDatabaseForeignKeys>; + : ValidateDatabaseRelationships>; function dumboDatabase( schemas: Schemas, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 57e6ac4e..a2c2b9ef 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import { SQL } from '../../sql'; import type { Equal, Expect } from '../../testing'; import type { TableColumnNames, TableRowType } from '../components'; -import { foreignKey } from '../components'; +import { relationship } from '../components'; import { dumboSchema } from './index'; const { database, schema, table, column, index } = dumboSchema; @@ -151,7 +151,7 @@ const users = table('users', { email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, - foreignKeys: [foreignKey(['id'], ['public.profiles.user_id'])], + relationships: [relationship(['id'], ['public.profiles.user_id'])], }); const _users2 = table('users', { @@ -160,7 +160,7 @@ const _users2 = table('users', { email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['id'], references: ['public.profiles.user_id'], @@ -195,7 +195,7 @@ const multiSchemaDb = database('myapp', { userId: column('user_id', Varchar('max')), timestamp: column('timestamp', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['userId'], references: ['public.users.id'], @@ -237,16 +237,16 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['user_id'], references: ['public.users.id'] }, ], }), }), }); - assert.ok(db.schemas.public.tables.posts.foreignKeys); + assert.ok(db.schemas.public.tables.posts.relationships); assert.deepStrictEqual( - db.schemas.public.tables.posts.foreignKeys[0].columns, + db.schemas.public.tables.posts.relationships[0].columns, ['user_id'], ); }); @@ -266,7 +266,7 @@ void describe('Foreign Key Validation', () => { user_id: column('user_id', Varchar('max')), tenant_id: column('tenant_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['user_id', 'tenant_id'], references: ['public.users.id', 'public.users.tenant_id'], @@ -277,7 +277,7 @@ void describe('Foreign Key Validation', () => { }); assert.deepStrictEqual( - db.schemas.public.tables.posts.foreignKeys[0].columns, + db.schemas.public.tables.posts.relationships[0].columns, ['user_id', 'tenant_id'], ); }); @@ -290,16 +290,16 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), manager_id: column('manager_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['manager_id'], references: ['public.users.id'] }, ] as const, }), }), }); - assert.ok(db.schemas.public.tables.users.foreignKeys); + assert.ok(db.schemas.public.tables.users.relationships); assert.deepStrictEqual( - db.schemas.public.tables.users.foreignKeys[0].references, + db.schemas.public.tables.users.relationships[0].references, ['public.users.id'], ); }); @@ -318,7 +318,7 @@ void describe('Foreign Key Validation', () => { user_id: column('user_id', Varchar('max')), author_id: column('author_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['user_id'], references: ['public.users.id'] }, { columns: ['author_id'], references: ['public.users.id'] }, ] as const, @@ -326,7 +326,7 @@ void describe('Foreign Key Validation', () => { }), }); - assert.strictEqual(db.schemas.public.tables.posts.foreignKeys.length, 2); + assert.strictEqual(db.schemas.public.tables.posts.relationships.length, 2); }); void it('should accept cross-schema foreign key', () => { @@ -344,7 +344,7 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - foreignKeys: [ + relationships: [ { columns: ['user_id'], references: ['public.users.id'] }, ], }), @@ -352,7 +352,7 @@ void describe('Foreign Key Validation', () => { }); assert.deepStrictEqual( - db.schemas.analytics.tables.events.foreignKeys[0].references, + db.schemas.analytics.tables.events.relationships[0].references, ['public.users.id'], ); }); diff --git a/src/packages/dumbo/src/core/schema/schema_feature.md b/src/packages/dumbo/src/core/schema/schema_feature.md index 9de5d3a3..4edf5f06 100644 --- a/src/packages/dumbo/src/core/schema/schema_feature.md +++ b/src/packages/dumbo/src/core/schema/schema_feature.md @@ -1176,7 +1176,7 @@ Support foreign keys across sharded databases: ```typescript // Define relationship that spans databases -const relationship = crossDatabaseForeignKey( +const relationship = crossDatabaseRelationship( 'read_models.reservations.guest_id', 'read_models.guests.id', { onDelete: 'CASCADE' }, diff --git a/todo.md b/todo.md index 5a351423..0604cb46 100644 --- a/todo.md +++ b/todo.md @@ -1,6 +1,7 @@ # Foreign Key Type Validation - Implementation Checklist ## Phase 1: Foundation - Type Extraction Utilities + - [x] Step 1: Add ExpectError Helper - [x] Step 2: Create Foreign Key Types File Structure - [ ] Step 3: Implement ExtractSchemaNames Type Utility @@ -8,31 +9,37 @@ - [ ] Step 5: Implement ExtractColumnNames Type Utility ## Phase 2: Column Reference Generation + - [ ] Step 6: Implement AllColumnReferences Type Utility (Single Schema) - [ ] Step 7: Test AllColumnReferences with Multi-Schema Database ## Phase 3: Foreign Key Type Definition -- [ ] Step 8: Define ForeignKeyDefinition Type + +- [ ] Step 8: Define RelationshipDefinition Type - [ ] Step 9: Update TableSchemaComponent to Include Foreign Keys - [ ] Step 10: Update dumboTable to Accept Foreign Keys ## Phase 4: Single Foreign Key Validation -- [ ] Step 11: Implement ValidateForeignKeyLength -- [ ] Step 12: Implement ValidateForeignKeyColumns -- [ ] Step 13: Implement ValidateForeignKeyReferences -- [ ] Step 14: Implement ValidateSingleForeignKey (Combine Validations) + +- [ ] Step 11: Implement ValidateRelationshipLength +- [ ] Step 12: Implement ValidateRelationshipColumns +- [ ] Step 13: Implement ValidateRelationshipReferences +- [ ] Step 14: Implement ValidateSingleRelationship (Combine Validations) ## Phase 5: Full Database Validation -- [ ] Step 15: Implement ValidateTableForeignKeys -- [ ] Step 16: Implement ValidateSchemaForeignKeys -- [ ] Step 17: Implement ValidateDatabaseForeignKeys + +- [ ] Step 15: Implement ValidateTableRelationships +- [ ] Step 16: Implement ValidateSchemaRelationships +- [ ] Step 17: Implement ValidateDatabaseRelationships ## Phase 6: Integration with dumboSchema + - [ ] Step 18: Create Foreign Keys Barrel Export - [ ] Step 19: Wire Validation to database() Function - [ ] Step 20: Wire Validation to schema() Function ## Phase 7: Real-World Testing + - [ ] Step 21: Add Real-World Test Cases - [ ] Step 22: Update Existing Example in Test File From ee1fa6f12892f5b7fb95fd2c23ea6a2a3abfe692 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 25 Nov 2025 11:57:33 +0100 Subject: [PATCH 38/39] Added relationship type --- .../relationships/relationshipTypes.ts | 14 ++ .../relationships/relationshipValidation.ts | 10 +- .../relationshipValidation.type.spec.ts | 160 +++++++++++++----- .../schema/components/tableSchemaComponent.ts | 22 +-- .../dumboSchema/dumboSchema.unit.spec.ts | 84 ++++++--- 5 files changed, 197 insertions(+), 93 deletions(-) diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts index d6e5fe3c..731402d9 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts @@ -47,21 +47,35 @@ export type AllColumnReferences = }[keyof Schemas] : never; +export type RelationshipType = + | 'one-to-one' + | 'one-to-many' + | 'many-to-one' + | 'many-to-many'; + export type RelationshipDefinition = { readonly columns: readonly Columns[]; readonly references: readonly References[]; + readonly type: RelationshipType; }; +export type TableRelationships = Record< + string, + RelationshipDefinition +>; + export const relationship = < const Columns extends readonly string[], const References extends readonly string[], >( columns: Columns, references: References, + type: RelationshipType, ) => { return { columns, references, + type, } as const; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts index c3b16bf5..c37a3ed0 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts @@ -111,14 +111,14 @@ export type ValidateSingleRelationship< ? ValidationResult : ValidationResult; -export type ValidateRelationshipArray< - FKs extends readonly AnyRelationshipDefinition[], +export type ValidateRelationship< + FKs extends Record, TableColumns extends string, ValidReferences extends string, -> = FKs extends readonly [] +> = keyof FKs extends never ? ValidationResult : ValidateSingleRelationship< - FKs[number], + FKs[keyof FKs], TableColumns, ValidReferences > extends { @@ -133,7 +133,7 @@ export type ValidateTableRelationships< ValidReferences extends string, > = Table extends TableSchemaComponent - ? ValidateRelationshipArray< + ? ValidateRelationship< FKs, TableColumnNames
& string, ValidReferences diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts index b8799cec..baa6fad8 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts @@ -140,6 +140,7 @@ type _Test9 = Expect< const _validFK: RelationshipDefinition = { columns: ['user_id'], references: ['public.users.id'], + type: 'one-to-one', }; type _ColumnsType = typeof _validFK.columns; @@ -148,6 +149,7 @@ type _Test10 = Expect>; const _compositeFK: RelationshipDefinition = { columns: ['user_id', 'tenant_id'], references: ['public.users.id', 'public.users.tenant_id'], + type: 'one-to-one', }; type _CompositeColumnsType = typeof _compositeFK.columns; @@ -312,28 +314,30 @@ type _Test24A = Expect< > >; -import type { ValidateRelationshipArray } from './relationshipValidation'; +import type { ValidateRelationship } from './relationshipValidation'; -type _FKArray_Mixed = readonly [ - { +type _FKRecord_Mixed = { + user_fk: { columns: ['user_id']; references: ['public.users.id']; - }, - { + type: 'one-to-many'; + }; + invalid_fk: { columns: ['invalid_col']; references: ['public.users.email']; - }, -]; + type: 'one-to-many'; + }; +}; -type _Result_FKArray_Mixed = ValidateRelationshipArray< - _FKArray_Mixed, +type _Result_FKRecord_Mixed = ValidateRelationship< + _FKRecord_Mixed, 'id' | 'user_id', 'public.users.id' | 'public.users.email' >; type _Test25A = Expect< Equal< - _Result_FKArray_Mixed, + _Result_FKRecord_Mixed, { valid: false; error: @@ -342,31 +346,39 @@ type _Test25A = Expect< } > >; -type _Test35 = Expect>; +type _Test35 = Expect>; -type _FKArray_AllValid = readonly [ - { +type _FKRecord_AllValid = { + user_fk: { columns: ['user_id']; references: ['public.users.id']; - }, - { + type: 'one-to-many'; + }; + email_fk: { columns: ['email']; references: ['public.users.email']; - }, -]; + type: 'one-to-many'; + }; +}; -type _Result_FKArray_AllValid = ValidateRelationshipArray< - _FKArray_AllValid, +type _Result_FKRecord_AllValid = ValidateRelationship< + _FKRecord_AllValid, 'id' | 'user_id' | 'email', 'public.users.id' | 'public.users.email' >; -type _ValidateRelationshipArrayResult_InvalidFK = ValidateRelationshipArray< - [{ columns: ['invalid']; references: ['public.users.id'] }], +type _ValidateRelationshipRecordResult_InvalidFK = ValidateRelationship< + { + invalid_fk: { + columns: ['invalid']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + }, 'id' | 'user_id', 'public.users.id' | 'public.users.email' >; -type _TestValidateRelationshipArrayResult_InvalidFK = Expect< - IsError<_ValidateRelationshipArrayResult_InvalidFK> +type _TestValidateRelationshipRecordResult_InvalidFK = Expect< + IsError<_ValidateRelationshipRecordResult_InvalidFK> >; import type { ValidateTableRelationships } from './relationshipValidation'; @@ -387,7 +399,13 @@ type _Table_SingleFK = TableSchemaComponent< id: AnyColumnSchemaComponent; user_id: AnyColumnSchemaComponent; }, - [{ columns: ['user_id']; references: ['public.users.id'] }] + { + author: { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + } >; type _Result_SingleFK = ValidateTableRelationships< @@ -402,10 +420,18 @@ type _Table_MultipleFK = TableSchemaComponent< user_id: AnyColumnSchemaComponent; author_id: AnyColumnSchemaComponent; }, - [ - { columns: ['user_id']; references: ['public.users.id'] }, - { columns: ['author_id']; references: ['public.users.id'] }, - ] + { + user: { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + author: { + columns: ['author_id']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + } >; type _Result_MultipleFK = ValidateTableRelationships< @@ -419,7 +445,13 @@ type _Table_InvalidFK = TableSchemaComponent< id: AnyColumnSchemaComponent; user_id: AnyColumnSchemaComponent; }, - [{ readonly columns: ['id']; references: readonly ['public.users.id'] }] + { + user: { + readonly columns: ['id']; + references: readonly ['public.users.id']; + type: 'one-to-many'; + }; + } >; type _Result_InvalidFK = ValidateTableRelationships< @@ -440,7 +472,13 @@ type _Schema_MultiTable = DatabaseSchemaSchemaComponent<{ id: AnyColumnSchemaComponent; user_id: AnyColumnSchemaComponent; }, - [{ columns: ['user_id']; references: ['public.users.id'] }] + { + user: { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + } >; }>; @@ -456,7 +494,13 @@ type _Schema_WithError = DatabaseSchemaSchemaComponent<{ id: AnyColumnSchemaComponent; user_id: AnyColumnSchemaComponent; }, - [{ columns: ['id']; references: ['public.users.id'] }] + { + user: { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-many'; + }; + } >; }>; @@ -473,7 +517,13 @@ const _dbWithErrorVSInDB = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: { + invalid: { + columns: ['id'], + references: ['public.users.id'], + type: 'one-to-many', + }, + }, }), }), }); @@ -495,9 +545,13 @@ const _fullDb = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [ - { columns: ['user_id'], references: ['public.users.id'] }, - ], + relationships: { + user: { + type: 'many-to-one', + columns: ['user_id'], + references: ['public.users.id'], + }, + }, }), }), }); @@ -512,9 +566,13 @@ const _dbWithSelfRef = database('test', { id: column('id', Varchar('max')), manager_id: column('manager_id', Varchar('max')), }, - relationships: [ - { columns: ['manager_id'], references: ['public.users.id'] }, - ], + relationships: { + manager: { + columns: ['manager_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); @@ -529,7 +587,13 @@ const _dbWithError = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: { + invalid: { + columns: ['id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); @@ -544,7 +608,13 @@ const _dbInvalidColumn = database('test', { id: column('id', Varchar('max')), user_id: column('id', Varchar('max')), }, - relationships: [{ columns: ['id'], references: ['public.users.id'] }], + relationships: { + invalid: { + columns: ['id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); @@ -563,9 +633,13 @@ const _dbValid = database('test', { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [ - { columns: ['user_id'], references: ['public.users.id'] }, - ], + relationships: { + user: { + columns: ['user_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index 8b0eb53c..deb496c5 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -12,7 +12,7 @@ import { IndexURNType, type IndexSchemaComponent, } from './indexSchemaComponent'; -import type { RelationshipDefinition } from './relationships/relationshipTypes'; +import type { TableRelationships } from './relationships/relationshipTypes'; import type { TableColumnNames } from './tableTypesInference'; export type TableURNType = 'sc:dumbo:table'; @@ -23,24 +23,11 @@ export const TableURN = ({ name }: { name: string }): TableURN => `${TableURNType}:${name}`; export type TableColumns = Record; -export type TableRelationships< - Columns extends string = string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - FKS extends RelationshipDefinition = RelationshipDefinition< - Columns, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - any - >, -> = - FKS extends RelationshipDefinition - ? readonly RelationshipDefinition[] - : never; export type TableSchemaComponent< Columns extends TableColumns = TableColumns, - Relationships extends TableRelationships< - keyof Columns & string - > = TableRelationships, + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + Relationships extends TableRelationships = {}, > = SchemaComponent< TableURN, Readonly<{ @@ -61,7 +48,8 @@ export type AnyTableSchemaComponent = TableSchemaComponent; export const tableSchemaComponent = < Columns extends TableColumns = TableColumns, - const Relationships extends TableRelationships = TableRelationships, + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + const Relationships extends TableRelationships = {}, >({ tableName, columns, diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index a2c2b9ef..6baff728 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -151,7 +151,9 @@ const users = table('users', { email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, - relationships: [relationship(['id'], ['public.profiles.user_id'])], + relationships: { + profile: relationship(['id'], ['public.profiles.user_id'], 'one-to-one'), + }, }); const _users2 = table('users', { @@ -160,12 +162,13 @@ const _users2 = table('users', { email: column('email', Varchar('max'), { notNull: true }), name: column('name', Varchar('max')), }, - relationships: [ - { + relationships: { + profile: { columns: ['id'], references: ['public.profiles.user_id'], + type: 'one-to-one', }, - ], + }, }); export const simpleDb = database( @@ -195,12 +198,13 @@ const multiSchemaDb = database('myapp', { userId: column('user_id', Varchar('max')), timestamp: column('timestamp', Varchar('max')), }, - relationships: [ - { + relationships: { + user: { columns: ['userId'], references: ['public.users.id'], + type: 'many-to-one', }, - ], + }, }), }), }); @@ -237,16 +241,20 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [ - { columns: ['user_id'], references: ['public.users.id'] }, - ], + relationships: { + user: { + columns: ['user_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); assert.ok(db.schemas.public.tables.posts.relationships); assert.deepStrictEqual( - db.schemas.public.tables.posts.relationships[0].columns, + db.schemas.public.tables.posts.relationships.user.columns, ['user_id'], ); }); @@ -266,18 +274,19 @@ void describe('Foreign Key Validation', () => { user_id: column('user_id', Varchar('max')), tenant_id: column('tenant_id', Varchar('max')), }, - relationships: [ - { + relationships: { + user: { columns: ['user_id', 'tenant_id'], references: ['public.users.id', 'public.users.tenant_id'], + type: 'many-to-one', }, - ], + }, }), }), }); assert.deepStrictEqual( - db.schemas.public.tables.posts.relationships[0].columns, + db.schemas.public.tables.posts.relationships.user.columns, ['user_id', 'tenant_id'], ); }); @@ -290,16 +299,20 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), manager_id: column('manager_id', Varchar('max')), }, - relationships: [ - { columns: ['manager_id'], references: ['public.users.id'] }, - ] as const, + relationships: { + manager: { + columns: ['manager_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + } as const, }), }), }); assert.ok(db.schemas.public.tables.users.relationships); assert.deepStrictEqual( - db.schemas.public.tables.users.relationships[0].references, + db.schemas.public.tables.users.relationships.manager.references, ['public.users.id'], ); }); @@ -318,15 +331,26 @@ void describe('Foreign Key Validation', () => { user_id: column('user_id', Varchar('max')), author_id: column('author_id', Varchar('max')), }, - relationships: [ - { columns: ['user_id'], references: ['public.users.id'] }, - { columns: ['author_id'], references: ['public.users.id'] }, - ] as const, + relationships: { + user: { + columns: ['user_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + author: { + columns: ['author_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + } as const, }), }), }); - assert.strictEqual(db.schemas.public.tables.posts.relationships.length, 2); + assert.strictEqual( + Object.keys(db.schemas.public.tables.posts.relationships).length, + 2, + ); }); void it('should accept cross-schema foreign key', () => { @@ -344,15 +368,19 @@ void describe('Foreign Key Validation', () => { id: column('id', Varchar('max')), user_id: column('user_id', Varchar('max')), }, - relationships: [ - { columns: ['user_id'], references: ['public.users.id'] }, - ], + relationships: { + user: { + columns: ['user_id'], + references: ['public.users.id'], + type: 'many-to-one', + }, + }, }), }), }); assert.deepStrictEqual( - db.schemas.analytics.tables.events.relationships[0].references, + db.schemas.analytics.tables.events.relationships.user.references, ['public.users.id'], ); }); From 3e35bff9bb40c774cf6f6561813161f8f1a5be82 Mon Sep 17 00:00:00 2001 From: Oskar Dudycz Date: Tue, 25 Nov 2025 12:41:33 +0100 Subject: [PATCH 39/39] Refactored relationship validation to include type validation and partial matches --- plan.md | 1619 --------------- .../core/schema/MIGRATION_UNIFICATION_PLAN.md | 1731 ----------------- src/packages/dumbo/src/core/schema/REVIEW.MD | 201 -- .../components/columnSchemaComponent.ts | 28 +- .../databaseSchemaSchemaComponent.ts | 26 +- .../collectReferencesErrors.type.spec.ts | 272 +++ .../collectRelationshipErrors.type.spec.ts | 346 ++++ .../relationships/formatRelationshipErrors.ts | 155 ++ .../formatRelationshipErrors.type.spec.ts | 404 ++++ .../schema/components/relationships/index.ts | 2 + .../relationships/relationshipTypes.ts | 229 ++- .../relationships/relationshipValidation.ts | 651 +++++-- .../relationshipValidation.type.spec.ts | 650 ------- .../validateDatabaseSchema.type.spec.ts | 187 ++ .../validateDatabaseSchemas.type.spec.ts | 190 ++ .../validateReference.type.spec.ts | 365 ++++ .../validateRelationship.type.spec.ts | 421 ++++ .../validateRelationshipColumns.type.spec.ts | 167 ++ .../validateRelationshipLength.type.spec.ts | 124 ++ .../validateTableRelationships.type.spec.ts | 236 +++ .../schema/components/tableSchemaComponent.ts | 42 +- .../schema/components/tableTypesInference.ts | 8 +- .../tableTypesInference.type.spec.ts | 42 +- .../dumbo/src/core/schema/dumboFeature.md | 1606 --------------- .../core/schema/dumboSchema/dumboSchema.ts | 65 +- .../dumboSchema/dumboSchema.unit.spec.ts | 10 +- .../dumbo/src/core/schema/schemaComponent.ts | 2 +- .../dumbo/src/core/schema/schema_feature.md | 1345 ------------- .../dumbo/src/core/testing/typesTesting.ts | 7 +- .../dumbo/src/core/typing/conditionals.ts | 43 + src/packages/dumbo/src/core/typing/index.ts | 4 + src/packages/dumbo/src/core/typing/records.ts | 4 + src/packages/dumbo/src/core/typing/tuples.ts | 148 ++ .../dumbo/src/core/typing/validation.ts | 66 + todo.md | 78 - 35 files changed, 3981 insertions(+), 7493 deletions(-) delete mode 100644 plan.md delete mode 100644 src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md delete mode 100644 src/packages/dumbo/src/core/schema/REVIEW.MD create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/collectReferencesErrors.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/collectRelationshipErrors.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.type.spec.ts delete mode 100644 src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchema.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchemas.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateReference.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateRelationship.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipColumns.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipLength.type.spec.ts create mode 100644 src/packages/dumbo/src/core/schema/components/relationships/validateTableRelationships.type.spec.ts delete mode 100644 src/packages/dumbo/src/core/schema/dumboFeature.md delete mode 100644 src/packages/dumbo/src/core/schema/schema_feature.md create mode 100644 src/packages/dumbo/src/core/typing/conditionals.ts create mode 100644 src/packages/dumbo/src/core/typing/index.ts create mode 100644 src/packages/dumbo/src/core/typing/records.ts create mode 100644 src/packages/dumbo/src/core/typing/tuples.ts create mode 100644 src/packages/dumbo/src/core/typing/validation.ts delete mode 100644 todo.md diff --git a/plan.md b/plan.md deleted file mode 100644 index f07b9008..00000000 --- a/plan.md +++ /dev/null @@ -1,1619 +0,0 @@ -# Foreign Key Type Validation Implementation Plan - -## Overview - -Implement compile-time type validation for foreign keys in Dumbo schema definitions. The validation happens at the database/schema level (top-down), ensuring all foreign key references are valid without requiring `as const` annotations. - -## Goals - -- ✅ Type-safe foreign key definitions with full intellisense -- ✅ Validate `columns` array contains only columns from the defining table -- ✅ Validate `references` array contains only valid `schema.table.column` paths -- ✅ Support composite foreign keys (multiple columns mapping to multiple references) -- ✅ Support self-referential foreign keys -- ✅ Clear, actionable error messages when validation fails -- ✅ No runtime validation - pure TypeScript type-level implementation -- ✅ No `as const` required - use generic inference - -## Technical Approach - -### Type System Strategy - -1. Use generic inference in `table()` function to capture exact tuple types from array literals -2. Store foreign key definitions in table schema component -3. Validate all foreign keys at `database()` level using recursive type utilities -4. Show all validation errors (optimize to first error only if performance issues arise) - -### Validation Flow - -``` -database() called - → Extract all schemas - → Extract all tables from each schema - → Extract all columns from each table - → Build union of all valid column references: 'schema.table.column' - → For each table with relationships: - → Validate each FK's columns array against table's columns - → Validate each FK's references array against all valid column references - → Validate columns.length === references.length - → Collect all errors with helpful messages - → If any errors: database() RETURN TYPE becomes ValidationResult - → If no errors: database() RETURN TYPE is DatabaseSchemaComponent - → TypeScript shows errors at database() call site when types don't match - → Use Expect/ExpectError type tests to verify validation works -``` - -**Why Return Types:** -TypeScript cannot validate function parameters before type inference completes. The solution is conditional return types - when validation fails, the return type becomes an error object (`{ valid: false; error: "..." }`) which is incompatible with `DatabaseSchemaComponent`, causing type errors at the assignment site. - -## Quality Gates - -**CRITICAL: After EVERY step, run the following as subagents (to reduce context):** - -1. `cd src & npm run fix` - Fix linting issues -2. `cd src & npm run build:ts` - Ensure TypeScript compiles -3. `cd src & npm run test:unit` - Run all tests - -YOU MUST run them through sub agents, not bash. - -**All three must pass before proceeding to the next step.** - -If any fail and cannot be fixed automatically, **STOP and ask for help**. - -## Validation Protocol - -For EACH step: - -1. Make changes to the code -2. Run quality gates ONCE via subagents in parallel: - - `npm run fix` - - `npm run build:ts` - - `npm run test:unit` -3. If any gate fails, fix the issue and trust that the fix worked (don't rerun) -4. Commit immediately after fixes -5. Mark step as complete - -NEVER re-run the same command twice. Trust subagent results. - -## Code Standards - -- Use `describe`/`it` for runtime tests (matching existing test patterns) -- For type tests (\*.type.spec.ts), use simple type assertions at top level (no `void it()` wrappers - see tableTypesInference.type.spec.ts) -- Never import js files, always use typescript, -- No classes unless absolutely necessary -- Avoid over-complexity -- No comments unless they add genuine value -- Follow existing codebase patterns and conventions -- Prefer simple, clean, maintainable solutions - -## Implementation Phases - -### Phase 1: Foundation - Type Extraction Utilities - -Build the base type utilities to extract schema structure information. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (new) -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (new) -- `src/packages/dumbo/src/core/testing/typesTesting.ts` (modify - add ExpectError) - -**Deliverables:** - -- `ExpectError` helper type -- `ExtractSchemaNames` - extract schema names from database -- `ExtractTableNames` - extract table names from schema -- Type tests for all extraction utilities (reuse existing `TableColumnNames` for columns) - -### Phase 2: Column Reference Generation - -Build utilities to generate all valid column reference paths. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (modify) -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) - -**Deliverables:** - -- `AllColumnReferences` - generate union of all valid 'schema.table.column' strings -- Type tests for single-schema and multi-schema databases - -### Phase 3: Foreign Key Type Definition - -Define the foreign key structure and add it to table schema component. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts` (modify) -- `src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts` (modify) -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) - -**Deliverables:** - -- `RelationshipDefinition` type -- Updated `TableSchemaComponent` with generic FK parameter -- Type tests for FK structure - -### Phase 4: Single Foreign Key Validation - -Implement validation logic for a single foreign key definition. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts` (new) -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) - -**Deliverables:** - -- `ValidateRelationshipColumns` - validate columns array -- `ValidateRelationshipReferences` - validate references array -- `ValidateRelationshipLength` - validate columns.length === references.length -- `ValidateSingleRelationship` - combine all validations -- Comprehensive error message types with helpful suggestions -- Type tests for valid and invalid scenarios - -### Phase 5: Full Database Validation - -Implement database-level validation that checks all foreign keys across all tables. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts` (modify) -- `src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts` (modify) - -**Deliverables:** - -- `ValidateTableRelationships` - validate all FKs in a table -- `ValidateSchemaRelationships` - validate all FKs in a schema -- `ValidateDatabaseRelationships` - validate all FKs in entire database -- Type tests for multi-table, multi-schema validation - -### Phase 6: Integration with dumboSchema - -Wire up validation to the `database()` and `schema()` functions. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts` (modify) -- `src/packages/dumbo/src/core/schema/components/index.ts` (modify - re-export FK types) -- `src/packages/dumbo/src/core/schema/relationships/index.ts` (new - barrel export) - -**Deliverables:** - -- Updated `dumboTable` signature to accept and capture `relationships` generic -- Updated `dumboDatabase` signature with FK validation constraint -- Updated `dumboDatabaseSchema` signature with FK validation constraint -- Proper type exports - -### Phase 7: Real-World Testing - -Test with actual schema definitions and ensure everything works. - -**Files to create/modify:** - -- `src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts` (modify) - -**Deliverables:** - -- Working examples with valid foreign keys -- Multiple test cases (single FK, composite FK, self-referential, multiple FKs, cross-schema) -- Update existing multiSchemaDb example to verify it works correctly - ---- - -## Detailed Step-by-Step Prompts - -Each prompt below is self-contained and builds on the previous work. They follow TDD principles: write type tests first, then implement the types to pass the tests. - ---- - -### Step 1: Add ExpectError Helper - -**Context:** We need a type helper to test that validation errors are properly generated. This helper will be used throughout our type tests. - -**Prompt:** - -```` -Add an `ExpectError` helper type to src/packages/dumbo/src/core/testing/typesTesting.ts. - -The helper should: -- Accept a type parameter T -- Return true if T extends { valid: false } -- Be used like: type Test = ExpectError - -Add it below the existing Expect and Equal helpers, and export it. - -Example usage: -```typescript -type InvalidCase = { valid: false, error: string }; -type Test = ExpectError; -```` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- ExpectError type added and exported -- No breaking changes to existing tests -- All quality gates pass - ---- - -### Step 2: Create Foreign Key Types File Structure - -**Context:** Set up the base file structure for foreign key validation types and tests. - -**Prompt:** -``` - -Create two new files: - -1. src/packages/dumbo/src/core/schema/relationships/relationshipTypes.ts - - - Add placeholder comment: `// Foreign key type definitions` - -2. src/packages/dumbo/src/core/schema/relationships/relationshipValidation.type.spec.ts - - Import Expect, Equal, ExpectError from '../../testing' - - Add comment placeholders for test sections: - - Schema Structure Extraction - - Column Reference Generation - - Foreign Key Definition Structure - - Foreign Key Validation - Valid Cases - - Foreign Key Validation - Invalid Cases - - Integration Tests - - Follow the pattern from tableTypesInference.type.spec.ts (simple type assertions, no describe/it blocks) - -Ensure the test file is recognized by the test runner. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Both files created with proper structure -- Test file runs without errors -- File structure follows existing patterns -- All quality gates pass - ---- - -### Step 3: Implement ExtractSchemaNames Type Utility - -**Context:** We need to extract schema names from a database type to build validation logic. - -**Prompt:** -``` - -Following TDD approach, add type tests then implement ExtractSchemaNames utility. - -In relationshipValidation.type.spec.ts, add under 'Schema Structure Extraction' comment section: - -```typescript -// Schema Structure Extraction - -// ExtractSchemaNames - single schema -type _DB1 = DatabaseSchemaComponent<{ - public: DatabaseSchemaSchemaComponent; -}>; -type _Test1 = Expect, "public">>; - -// ExtractSchemaNames - multi schema -type _DB2 = DatabaseSchemaComponent<{ - public: DatabaseSchemaSchemaComponent; - analytics: DatabaseSchemaSchemaComponent; -}>; -type _Test2 = Expect, "public" | "analytics">>; -``` - -Then in relationshipTypes.ts, implement: - -```typescript -import type { DatabaseSchemaComponent, DatabaseSchemas } from "../components"; - -export type ExtractSchemaNames = - DB extends DatabaseSchemaComponent - ? keyof Schemas & string - : never; -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ExtractSchemaNames correctly extracts schema names -- All quality gates pass - ---- - -### Step 4: Implement ExtractTableNames Type Utility - -**Context:** Extract table names from a schema to build validation logic. - -**Prompt:** -``` - -Following TDD approach, add type tests then implement ExtractTableNames utility. - -In relationshipValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: - -```typescript -void it("should extract table names from single-table schema", () => { - type Schema = DatabaseSchemaSchemaComponent<{ - users: TableSchemaComponent; - }>; - - type Result = ExtractTableNames; - type Test = Expect>; -}); - -void it("should extract table names from multi-table schema", () => { - type Schema = DatabaseSchemaSchemaComponent<{ - users: TableSchemaComponent; - posts: TableSchemaComponent; - comments: TableSchemaComponent; - }>; - - type Result = ExtractTableNames; - type Test = Expect>; -}); -``` - -Then in relationshipTypes.ts, implement: - -```typescript -import type { - DatabaseSchemaSchemaComponent, - DatabaseSchemaTables, -} from "../components"; - -export type ExtractTableNames = - Schema extends DatabaseSchemaSchemaComponent< - infer Tables extends DatabaseSchemaTables - > - ? keyof Tables & string - : never; -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ExtractTableNames correctly extracts table names -- All quality gates pass - ---- - -### Step 5: Implement ExtractColumnNames Type Utility - -**Context:** Extract column names from a table to validate foreign key columns array. - -**Prompt:** -``` - -Following TDD approach, add type tests then implement ExtractColumnNames utility. - -In relationshipValidation.type.spec.ts, add to 'Schema Structure Extraction' describe block: - -```typescript -void it("should extract column names from single-column table", () => { - type Table = TableSchemaComponent<{ - id: AnyColumnSchemaComponent; - }>; - - type Result = ExtractColumnNames
; - type Test = Expect>; -}); - -void it("should extract column names from multi-column table", () => { - type Table = TableSchemaComponent<{ - id: AnyColumnSchemaComponent; - email: AnyColumnSchemaComponent; - name: AnyColumnSchemaComponent; - created_at: AnyColumnSchemaComponent; - }>; - - type Result = ExtractColumnNames
; - type Test = Expect>; -}); -``` - -Then in relationshipTypes.ts, implement: - -```typescript -import type { - TableSchemaComponent, - TableColumns, - TableColumnNames, -} from "../components"; - -export type ExtractColumnNames
= - Table extends TableSchemaComponent - ? TableColumnNames
- : never; -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ExtractColumnNames reuses existing TableColumnNames utility -- All quality gates pass - ---- - -### Step 6: Implement AllColumnReferences Type Utility (Single Schema) - -**Context:** Generate all valid column reference paths in 'schema.table.column' format. Start with single schema case. - -**Prompt:** -``` - -Following TDD approach, add type test then implement AllColumnReferences for single schema. - -In relationshipValidation.type.spec.ts, add to 'Column Reference Generation' describe block: - -```typescript -import { SQL } from "../../../sql"; -import { dumboSchema } from "../../dumboSchema"; - -const { database, schema, table, column } = dumboSchema; -const { Varchar } = SQL.column.type; - -void it("should generate column references for single schema with one table", () => { - const db = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - email: column("email", Varchar("max")), - }, - }), - }), - }); - - type Result = AllColumnReferences; - type Test = Expect>; -}); - -void it("should generate column references for single schema with multiple tables", () => { - const db = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - email: column("email", Varchar("max")), - }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - title: column("title", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - }), - }), - }); - - type Result = AllColumnReferences; - type Test = Expect< - Equal< - Result, - | "public.users.id" - | "public.users.email" - | "public.posts.id" - | "public.posts.title" - | "public.posts.user_id" - > - >; -}); -``` - -Then in relationshipTypes.ts, implement: - -```typescript -export type AllColumnReferences = - DB extends DatabaseSchemaComponent - ? { - [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< - infer Tables - > - ? { - [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< - infer Columns - > - ? { - [ColumnName in keyof Columns]: `${SchemaName & - string}.${TableName & string}.${ColumnName & string}`; - }[keyof Columns] - : never; - }[keyof Tables] - : never; - }[keyof Schemas] - : never; -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- AllColumnReferences generates correct paths for single schema -- All quality gates pass - ---- - -### Step 7: Test AllColumnReferences with Multi-Schema Database - -**Context:** Verify AllColumnReferences works with multiple schemas. - -**Prompt:** -``` - -Add type test for multi-schema database to verify AllColumnReferences works correctly. - -In relationshipValidation.type.spec.ts, add to 'Column Reference Generation' describe block: - -```typescript -void it("should generate column references for multi-schema database", () => { - const db = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - email: column("email", Varchar("max")), - }, - }), - }), - analytics: schema("analytics", { - events: table("events", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - event_type: column("event_type", Varchar("max")), - }, - }), - }), - }); - - type Result = AllColumnReferences; - type Test = Expect< - Equal< - Result, - | "public.users.id" - | "public.users.email" - | "analytics.events.id" - | "analytics.events.user_id" - | "analytics.events.event_type" - > - >; -}); -``` - -Run tests to ensure they pass. No implementation changes needed - this verifies existing implementation works. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type test added and passes -- AllColumnReferences works with multiple schemas -- All quality gates pass - ---- - -### Step 8: Define RelationshipDefinition Type - -**Context:** Define the structure for foreign key definitions that tables will use. - -**Prompt:** -``` - -Add the RelationshipDefinition type to relationshipTypes.ts and create a basic type test. - -In relationshipTypes.ts, add: - -```typescript -export type RelationshipDefinition = { - readonly columns: readonly string[]; - readonly references: readonly string[]; -}; -``` - -In relationshipValidation.type.spec.ts, add to 'Foreign Key Definition Structure' describe block: - -```typescript -void it("should accept valid foreign key definition", () => { - type FK = RelationshipDefinition; - - const validFK: FK = { - columns: ["user_id"], - references: ["public.users.id"], - }; - - type ColumnsType = typeof validFK.columns; - type Test = Expect>; -}); - -void it("should accept composite foreign key definition", () => { - type FK = RelationshipDefinition; - - const compositeFK: FK = { - columns: ["user_id", "tenant_id"], - references: ["public.users.id", "public.users.tenant_id"], - }; - - type ColumnsType = typeof compositeFK.columns; - type ReferencesType = typeof compositeFK.references; - type Test1 = Expect>; - type Test2 = Expect>; -}); -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- RelationshipDefinition type added -- Basic structure tests pass -- All quality gates pass - ---- - -### Step 9: Update TableSchemaComponent to Include Foreign Keys - -**Context:** Add foreign keys support to the table schema component type definition. - -**Prompt:** -``` - -Update TableSchemaComponent to support an optional relationships property. - -In src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts: - -1. Import RelationshipDefinition: - -```typescript -import type { RelationshipDefinition } from "../relationships/relationshipTypes"; -``` - -2. Add generic parameter for foreign keys and relationships property: - -```typescript -export type TableSchemaComponent< - Columns extends TableColumns = TableColumns, - Relationships extends readonly RelationshipDefinition[] = readonly RelationshipDefinition[] -> = SchemaComponent< - TableURN, - Readonly<{ - tableName: string; - columns: ReadonlyMap & Columns; - primaryKey: TableColumnNames< - TableSchemaComponent - >[]; - relationships?: Relationships; - indexes: ReadonlyMap; - addColumn: (column: AnyColumnSchemaComponent) => AnyColumnSchemaComponent; - addIndex: (index: IndexSchemaComponent) => IndexSchemaComponent; - }> ->; -``` - -3. Update AnyTableSchemaComponent: - -```typescript -export type AnyTableSchemaComponent = TableSchemaComponent; -``` - -4. Update tableSchemaComponent function signature and implementation to accept and return relationships. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- TableSchemaComponent updated with relationships support -- Generic parameter added for type safety -- No breaking changes to existing code -- All quality gates pass - ---- - -### Step 10: Update dumboTable to Accept Foreign Keys - -**Context:** Update the dumboTable function to accept and capture foreign keys with generic inference. - -**Prompt:** -``` - -Update dumboTable function in dumboSchema.ts to accept relationships parameter. - -In src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts: - -1. Import RelationshipDefinition at the top: - -```typescript -import type { RelationshipDefinition } from "../relationships/relationshipTypes"; -``` - -2. Update dumboTable function to accept relationships in definition parameter and pass it to tableSchemaComponent. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- dumboTable accepts relationships parameter -- Generic inference captures exact FK types -- No breaking changes -- All quality gates pass - ---- - -### Step 11: Implement ValidateRelationshipLength - -**Context:** Validate that foreign key columns and references arrays have matching lengths. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateRelationshipLength. - -Create new file src/packages/dumbo/src/core/schema/relationships/relationshipValidation.ts with validation result types and ValidateRelationshipLength. - -In relationshipValidation.type.spec.ts, add to 'Foreign Key Validation - Invalid Cases' describe block: - -```typescript -void it("should error when columns and references have different lengths", () => { - type FK = { - columns: ["user_id", "tenant_id"]; - references: ["public.users.id"]; - }; - - type Result = ValidateRelationshipLength; - type Test = ExpectError; -}); - -void it("should pass when columns and references have same length", () => { - type FK1 = { - columns: ["user_id"]; - references: ["public.users.id"]; - }; - - type FK2 = { - columns: ["user_id", "tenant_id"]; - references: ["public.users.id", "public.users.tenant_id"]; - }; - - type Result1 = ValidateRelationshipLength; - type Result2 = ValidateRelationshipLength; - type Test1 = Expect>; - type Test2 = Expect>; -}); -``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateRelationshipLength correctly validates array lengths -- All quality gates pass - ---- - -### Step 12: Implement ValidateRelationshipColumns - -**Context:** Validate that all columns in a foreign key exist in the table. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateRelationshipColumns. - -In relationshipValidation.type.spec.ts, add tests to both valid and invalid cases describe blocks. - -In relationshipValidation.ts, implement ValidateRelationshipColumns with helper types to check if all elements of a tuple are in a union and to find invalid columns. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateRelationshipColumns correctly validates column existence -- Helpful error messages show which columns are invalid -- All quality gates pass - ---- - -### Step 13: Implement ValidateRelationshipReferences - -**Context:** Validate that all references in a foreign key point to valid schema.table.column paths. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateRelationshipReferences. - -In relationshipValidation.type.spec.ts, add tests for valid and invalid reference scenarios. - -In relationshipValidation.ts, implement ValidateRelationshipReferences with helper to find invalid references. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateRelationshipReferences correctly validates reference paths -- Helpful error messages show invalid references and available options -- All quality gates pass - ---- - -### Step 14: Implement ValidateSingleRelationship (Combine Validations) - -**Context:** Combine all FK validations into a single validation function. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateSingleRelationship that combines all validations. - -In relationshipValidation.type.spec.ts, add tests for complete FK validation covering valid cases and all error scenarios. - -In relationshipValidation.ts, implement ValidateSingleRelationship that chains length, columns, and references validation. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateSingleRelationship checks all validation rules in order -- Returns first error encountered (fail fast) -- All quality gates pass - ---- - -### Step 15: Implement ValidateTableRelationships - -**Context:** Validate all foreign keys defined in a single table. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateTableRelationships. - -In relationshipValidation.type.spec.ts, add tests for tables with no FKs, single FK, and multiple FKs. - -In relationshipValidation.ts, implement ValidateTableRelationships with helper to iterate through FK array. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateTableRelationships handles tables with 0, 1, or multiple FKs -- Returns first error encountered across all FKs -- All quality gates pass - ---- - -### Step 16: Implement ValidateSchemaRelationships - -**Context:** Validate all foreign keys across all tables in a schema. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateSchemaRelationships. - -In relationshipValidation.type.spec.ts, add tests for schemas with multiple tables and FKs. - -In relationshipValidation.ts, implement ValidateSchemaRelationships with helper to iterate through tables. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateSchemaRelationships iterates through all tables -- Returns first error encountered -- All quality gates pass - ---- - -### Step 17: Implement ValidateDatabaseRelationships - -**Context:** Validate all foreign keys across all schemas in the database - the top-level validation. - -**Prompt:** -``` - -Following TDD, add type tests then implement ValidateDatabaseRelationships. - -In relationshipValidation.type.spec.ts, add to 'Integration Tests' describe block tests for complete database validation including self-referential FKs. - -In relationshipValidation.ts, implement ValidateDatabaseRelationships that validates all schemas using AllColumnReferences. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Type tests added and pass -- ValidateDatabaseRelationships validates entire database -- Handles self-referential FKs correctly -- Returns first error encountered -- All quality gates pass - ---- - -### Step 18: Create Foreign Keys Barrel Export - -**Context:** Create an index file to export all foreign key types cleanly. - -**Prompt:** -``` - -Create src/packages/dumbo/src/core/schema/relationships/index.ts to export all foreign key types. - -Update src/packages/dumbo/src/core/schema/components/index.ts to re-export FK types. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -```` - -**Acceptance Criteria:** -- Barrel export file created -- All FK types exported from components/index.ts -- No circular dependency issues -- All quality gates pass - ---- - -### Step 19: Wire Validation to database() Function - -**Context:** Add conditional return types to database() so invalid FKs cause type errors at the call site. - -**Implementation Strategy:** - -Use conditional return types on `database()` function overloads: - -```typescript -function dumboDatabase(schemas: Schemas): - ValidateDatabaseRelationships> extends { valid: true } - ? DatabaseSchemaComponent - : ValidateDatabaseRelationships>; -```` - -**How it works:** - -1. User calls `const db = database({...})` -2. TypeScript infers exact `Schemas` type from literal -3. Return type is evaluated: - - If validation passes: returns `DatabaseSchemaComponent` - - If validation fails: returns `{ valid: false; error: "..." }` -4. Error object is incompatible with `DatabaseSchemaComponent` -5. Type error appears at `const db = database(...)` line -6. Error message shows which validation failed - -**No `as const` needed** - Generic inference on `table()` already captures exact tuple types from array literals. - -**Prompt:** - -```` - -IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. - -## Part 1: Add Failing Type Tests - -Add to relationshipValidation.type.spec.ts: - -```typescript -// TEST: Invalid column should cause type error at database() call -const _dbInvalidColumn = database('test', { - public: schema('public', { - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('id', Varchar('max')), - }, - relationships: [ - { columns: ['invalid_col'], references: ['public.users.id'] }, - ], - }), - }), -}); - -type _InvalidColResult = typeof _dbInvalidColumn; -type _Test_InvalidColumn = Expect< - Equal<_InvalidColResult, DatabaseSchemaComponent> ->; // This should FAIL because _InvalidColResult is error type - -// TEST: Valid FK should work -const _dbValid = database('test', { - public: schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, - }), - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - relationships: [{ columns: ['user_id'], references: ['public.users.id'] }], - }), - }), -}); - -type _ValidResult = typeof _dbValid; -type _Test_Valid = Expect< - Equal<_ValidResult, DatabaseSchemaComponent> ->; // This should PASS -```` - -Run `npm run build:ts` - you should see errors because validation not wired yet. - -## Part 2: Implement Validation - -Update dumboDatabase function in dumboSchema.ts to add FK validation via conditional return types. - -1. Import ValidateDatabaseRelationships from '../relationships' - -2. Find the TWO overload signatures that accept `schemas: Schemas` parameter (around lines 137 and 147) - - - Do NOT modify overloads that accept single `schema` parameter - -3. Change their return type from: - - ```typescript - DatabaseSchemaComponent; - ``` - - To: - - ```typescript - ValidateDatabaseRelationships> extends { valid: true } - ? DatabaseSchemaComponent - : ValidateDatabaseRelationships> - ``` - -4. Update the implementation function's return statement (around line 197) to cast the result: - ```typescript - return databaseSchemaComponent({ - databaseName, - schemas: schemaMap as Schemas, - ...dbOptions, - }) as ValidateDatabaseRelationships< - DatabaseSchemaComponent - > extends { - valid: true; - } - ? DatabaseSchemaComponent - : ValidateDatabaseRelationships>; - ``` - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Conditional return type added to database() overloads that accept schemas -- Type errors appear AT THE database() CALL SITE when invalid FKs provided -- Error messages show which FK validation failed -- Valid schemas continue to work with no changes -- All quality gates pass - ---- - -### Step 20: Wire Validation to schema() Function - -**Context:** Add FK validation to schema() for foreign keys that reference tables within the same schema. Cross-schema references cannot be validated at this level. - -**Implementation Strategy:** - -Similar to database(), use conditional return types on `schema()` function overloads. However, validation is limited to: -- FK columns must exist in the table -- FK references must be within the SAME schema (format: `schema.table.column` where schema matches current schema name) -- Cross-schema references will only be validated at database() level - -**Prompt:** -``` - -IMPORTANT: Follow TDD - Write failing type tests FIRST, then implement. - -## Part 1: Add Failing Type Tests - -Add to relationshipValidation.type.spec.ts: - -```typescript -// TEST: Invalid intra-schema FK at schema() level -const _schemaInvalidFK = schema("public", { - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [ - { columns: ["invalid_col"], references: ["public.users.id"] }, - ], - }), -}); - -type _SchemaInvalidResult = typeof _schemaInvalidFK; -type _Test_SchemaInvalid = Expect< - Equal<_SchemaInvalidResult, DatabaseSchemaSchemaComponent> ->; // Should FAIL - -// TEST: Valid intra-schema FK -const _schemaValidFK = schema("public", { - users: table("users", { - columns: { id: column("id", Varchar("max")) }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [{ columns: ["user_id"], references: ["public.users.id"] }], - }), -}); - -type _SchemaValidResult = typeof _schemaValidFK; -type _Test_SchemaValid = Expect< - Equal<_SchemaValidResult, DatabaseSchemaSchemaComponent> ->; // Should PASS -``` - -Run `npm run build:ts` - should see errors because schema validation not wired yet. - -## Part 2: Implement Validation - -Update dumboDatabaseSchema function in dumboSchema.ts to add FK validation for intra-schema references. - -1. Import ValidateSchemaRelationships from '../relationships' - -2. Find the TWO overload signatures that accept `tables: Tables` parameter - -3. Change their return type from: - - ```typescript - DatabaseSchemaSchemaComponent; - ``` - - To: - - ```typescript - ValidateSchemaRelationships< - DatabaseSchemaSchemaComponent, - AllColumnReferences> - > extends { valid: true } - ? DatabaseSchemaSchemaComponent - : ValidateSchemaRelationships< - DatabaseSchemaSchemaComponent, - AllColumnReferences> - > - ``` - -4. Update the implementation function's return statement to cast the result similarly - -Note: Schema-level validation only catches intra-schema FK errors. Cross-schema FKs are validated at database() level (Step 19). - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- Conditional return type added to schema() overloads -- Intra-schema FK errors caught at schema() call site -- Cross-schema FKs pass schema validation (caught later at database level) -- Database-level validation still primary mechanism -- All quality gates pass - ---- - -### Step 21: Add Real-World Test Cases - -**Context:** Test the validation with real schema definitions in dumboSchema.unit.spec.ts. - -**Prompt:** -``` - -Add real-world test cases to dumboSchema.unit.spec.ts demonstrating FK validation. - -Add a new describe block 'Foreign Key Validation' with test cases for: - -- Valid single FK -- Valid composite FK -- Self-referential FK -- Multiple FKs in one table -- Cross-schema FK - -Each test should verify the FK data is stored correctly in the schema component. - -After implementation, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass before proceeding. - -``` - -**Acceptance Criteria:** -- All real-world test cases added -- Tests verify FK data is stored correctly -- Tests demonstrate various FK scenarios -- All quality gates pass - ---- - -### Step 22: Add Comprehensive Compile-Time Validation Tests - -**Context:** Create type-level tests that verify all FK validation scenarios work at compile time. - -**Prompt:** -``` - -Add comprehensive type-level validation tests to relationshipValidation.type.spec.ts (after the basic type utility tests). - -Add these test cases demonstrating compile-time FK validation: - -```typescript -// ============================================================================ -// COMPILE-TIME FK VALIDATION TESTS -// ============================================================================ - -import { dumboSchema } from "../dumboSchema/dumboSchema"; -const { database, schema, table, column } = dumboSchema; - -// TEST 1: Invalid column name - should be error type -const _dbInvalidColumn = database("test", { - public: schema("public", { - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [ - { columns: ["nonexistent_col"], references: ["public.users.id"] }, - ], - }), - }), -}); - -type _InvalidColType = typeof _dbInvalidColumn; -// When validation fails, type should NOT equal DatabaseSchemaComponent -type _Test_InvalidColumn = Expect< - Equal<_InvalidColType, DatabaseSchemaComponent> -> extends true - ? never - : true; // Should be true (types don't match) - -// TEST 2: Invalid reference path - should be error type -const _dbInvalidRef = database("test", { - public: schema("public", { - users: table("users", { - columns: { id: column("id", Varchar("max")) }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [ - { columns: ["user_id"], references: ["public.nonexistent.id"] }, - ], - }), - }), -}); - -type _InvalidRefType = typeof _dbInvalidRef; -type _Test_InvalidRef = Expect< - Equal<_InvalidRefType, DatabaseSchemaComponent> -> extends true - ? never - : true; // Should be true - -// TEST 3: Length mismatch - should be error type -const _dbLengthMismatch = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - tenant_id: column("tenant_id", Varchar("max")), - }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - tenant_id: column("tenant_id", Varchar("max")), - }, - relationships: [ - { columns: ["user_id", "tenant_id"], references: ["public.users.id"] }, - ], - }), - }), -}); - -type _LengthMismatchType = typeof _dbLengthMismatch; -type _Test_LengthMismatch = Expect< - Equal<_LengthMismatchType, DatabaseSchemaComponent> -> extends true - ? never - : true; // Should be true - -// TEST 4: Valid FK - should work perfectly (type matches) -const _dbValidFK = database("test", { - public: schema("public", { - users: table("users", { - columns: { id: column("id", Varchar("max")) }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [ - { columns: ["user_id"], references: ["public.users.id"] }, - ], - }), - }), -}); - -type _ValidFKType = typeof _dbValidFK; -type _Test_ValidFK = Expect>>; // Should PASS (types match) - -// TEST 5: Composite FK - should work -const _dbCompositeFK = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - tenant_id: column("tenant_id", Varchar("max")), - }, - }), - posts: table("posts", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - tenant_id: column("tenant_id", Varchar("max")), - }, - relationships: [ - { - columns: ["user_id", "tenant_id"], - references: ["public.users.id", "public.users.tenant_id"], - }, - ], - }), - }), -}); - -type _CompositeFKType = typeof _dbCompositeFK; -type _Test_CompositeFK = Expect< - Equal<_CompositeFKType, DatabaseSchemaComponent> ->; // Should PASS - -// TEST 6: Self-referential FK - should work -const _dbSelfRef = database("test", { - public: schema("public", { - users: table("users", { - columns: { - id: column("id", Varchar("max")), - manager_id: column("manager_id", Varchar("max")), - }, - relationships: [ - { columns: ["manager_id"], references: ["public.users.id"] }, - ], - }), - }), -}); - -type _SelfRefType = typeof _dbSelfRef; -type _Test_SelfRef = Expect>>; // Should PASS - -// TEST 7: Cross-schema FK - should work -const _dbCrossSchema = database("test", { - public: schema("public", { - users: table("users", { - columns: { id: column("id", Varchar("max")) }, - }), - }), - analytics: schema("analytics", { - events: table("events", { - columns: { - id: column("id", Varchar("max")), - user_id: column("user_id", Varchar("max")), - }, - relationships: [ - { columns: ["user_id"], references: ["public.users.id"] }, - ], - }), - }), -}); - -type _CrossSchemaType = typeof _dbCrossSchema; -type _Test_CrossSchema = Expect< - Equal<_CrossSchemaType, DatabaseSchemaComponent> ->; // Should PASS -``` - -After adding tests, run quality gates as subagents: - -1. npm run fix -2. npm run build:ts -3. npm run test:unit - -All must pass. The invalid FK tests (1-3) should show type errors but still compile (they're type-level tests). - -Also verify the existing multiSchemaDb example (around line 163 in dumboSchema.unit.spec.ts) still works. - -``` - -**Acceptance Criteria:** -- All 7 type-level test scenarios added -- Invalid FKs (tests 1-3) show they DON'T match DatabaseSchemaComponent type -- Valid FKs (tests 4-7) show they DO match DatabaseSchemaComponent type -- Existing multiSchemaDb example compiles without errors -- All quality gates pass - ---- - -## Summary - -This implementation plan provides 22 detailed, step-by-step prompts to implement type-safe foreign key validation in the Dumbo schema system. Each step: - -- Follows TDD principles (tests first, implementation second) -- Builds incrementally on previous work -- Has clear acceptance criteria -- Runs quality gates (fix, build, test) as subagents after each step -- Is self-contained and executable -- No orphaned code - everything integrates - -The final result will be: -- ✅ Compile-time validation of foreign keys -- ✅ No `as const` required -- ✅ Support for composite and self-referential FKs -- ✅ Clear, actionable error messages -- ✅ Fully tested with type tests and unit tests - -Total estimated implementation time: 4-6 hours for an experienced TypeScript developer. -``` diff --git a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md b/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md deleted file mode 100644 index 33e3b756..00000000 --- a/src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md +++ /dev/null @@ -1,1731 +0,0 @@ -# Schema Component Migration Unification Plan - -## Overview - -This plan details the unification of SQL migrations and TypeScript schema definitions in the Dumbo schema component system. The goal is to support both SQL-first and TypeScript-first workflows while maintaining a clear source of truth and enabling strongly-typed query builders for Pongo. - -## Context and Current State - -### Key Files - -- **Core Schema Component**: [src/packages/dumbo/src/core/schema/schemaComponent.ts](src/packages/dumbo/src/core/schema/schemaComponent.ts) - Lines 3-146 -- **Component Implementations**: - - [src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts) - - [src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts) - - [src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts) - - [src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts) - - [src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts) -- **Migration System**: - - [src/packages/dumbo/src/core/schema/sqlMigration.ts](src/packages/dumbo/src/core/schema/sqlMigration.ts) - - [src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts](src/packages/dumbo/src/core/schema/migrators/schemaComponentMigrator.ts) - - [src/packages/dumbo/src/core/schema/migrators/migrator.ts](src/packages/dumbo/src/core/schema/migrators/migrator.ts) -- **Builder API**: [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts) -- **Tests**: [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts) -- **Pongo Integration Target**: [src/packages/pongo/src/core/schema/index.ts](src/packages/pongo/src/core/schema/index.ts) - -### Current Problem - -The system cannot distinguish between: - -1. **Initial schema definitions** (CREATE TABLE) -2. **Schema evolution** (ALTER TABLE) -3. **Data migrations** (UPDATE/INSERT) - -All migrations are stored in a single `migrations` array and bubble up through the component hierarchy (lines 79-83 in schemaComponent.ts), making it impossible to know the intent or current state. - -## Agreed Solution - -### Core Principle: TypeScript as Source of Truth (When Present) - -**The Rule**: When TypeScript definitions exist, they represent the desired schema state. Migrations are either: - -1. **Generated** from TypeScript definitions (if no migrations provided) -2. **Validated** against TypeScript definitions (if migrations provided) -3. **Trusted** completely (if no TypeScript definitions provided - SQL-first mode) - -### Three Usage Modes - -#### 1. TypeScript-First (Generates Migrations) - -```typescript -const table = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }), - }, - // No migrations = will generate CREATE TABLE IF NOT EXISTS -}); -``` - -#### 2. SQL-First (Migrations as Source of Truth) - -```typescript -const table = tableSchemaComponent({ - tableName: 'users', - migrations: [sqlMigration('001_create', [SQL`CREATE TABLE users ...`])], - // No columns = trust the SQL completely -}); -``` - -#### 3. Hybrid with Validation - -```typescript -const table = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial' }), - email: column('email', { type: 'varchar' }), - }, - migrations: [sqlMigration('001_create', [SQL`CREATE TABLE users ...`])], - // Both provided = TypeScript is truth, validate migrations match -}); -``` - -## Implementation Plan - -### Phase 1: Enhanced Column Definition Types - -**File to modify**: [src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts) - -```typescript -export type ColumnDefinition = { - type: - | 'serial' - | 'varchar' - | 'int' - | 'bigint' - | 'text' - | 'timestamp' - | 'boolean' - | 'uuid' - | 'json' - | 'jsonb'; - nullable?: boolean; - primaryKey?: boolean; - unique?: boolean; - length?: number; - precision?: number; - scale?: number; - defaultValue?: unknown; - references?: { - table: string; - column: string; - onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; - onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; - }; -}; - -export type ColumnSchemaComponent = SchemaComponent< - ColumnURN, - Readonly<{ - columnName: string; - definition?: ColumnDefinition; // NEW: Optional definition for TypeScript-first - generateMigration?: () => SQL; // NEW: Self-contained migration generation - }> ->; - -// Column knows how to generate its own SQL -export const generateColumnSQL = (column: ColumnSchemaComponent): string => { - if (!column.definition) return `${column.columnName} TEXT`; // fallback - - const { type, nullable, primaryKey, unique, length, defaultValue } = - column.definition; - let sql = `${column.columnName} ${type.toUpperCase()}`; - - if (length) sql += `(${length})`; - if (primaryKey) sql += ' PRIMARY KEY'; - if (unique) sql += ' UNIQUE'; - if (!nullable) sql += ' NOT NULL'; - if (defaultValue !== undefined) sql += ` DEFAULT ${defaultValue}`; - - return sql; -}; -``` - -### Phase 2: Nested Schema Snapshot System - -**Complete directory structure**: - -``` -.dumbo/ - snapshots/ # Current state + migration tracking - myapp.snapshot.ts # Root database snapshot - schemas/ - public.snapshot.ts # Schema-level snapshot - analytics.snapshot.ts - public/ - users.snapshot.ts # Table-level snapshots - posts.snapshot.ts - comments.snapshot.ts - analytics/ - events.snapshot.ts - metrics.snapshot.ts - migrations/ # All SQL migrations (generated or provided) - 2024-01-15-001-create_schema_public.sql - 2024-01-15-002-create_table_users.sql - 2024-01-15-003-create_index_idx_users_email.sql - 2024-01-16-001-add_email_to_users.sql -``` - -Note: Applied migrations are tracked in the database `migrations` table (see `migrator.ts` lines 167-189), not in files. - -**Migration SQL file examples**: - -```sql --- .dumbo/migrations/2024-01-15-002-create_table_users.sql --- Source: generated (from TypeScript definitions) --- Component: sc:dumbo:table:users --- Generated at: 2024-01-15T10:30:00Z -CREATE TABLE IF NOT EXISTS users ( - id SERIAL PRIMARY KEY, - email VARCHAR(255) NOT NULL -); - --- .dumbo/migrations/2024-01-16-001-custom_users_setup.sql --- Source: provided (from component migrations array) --- Component: sc:dumbo:table:users --- Migration name: 001_custom_setup -CREATE TABLE users ( - id SERIAL PRIMARY KEY, - email VARCHAR(255), - internal_field JSONB -); -``` - -**Snapshot file examples with migration tracking**: - -```typescript -// SCENARIO 1: TypeScript-first (no migrations provided) -// Component: tableSchemaComponent({ -// tableName: 'users', -// columns: { id: column('id', { type: 'serial', primaryKey: true }) } -// }) - -// .dumbo/snapshots/schemas/public/users.snapshot.ts -export const usersSnapshot = { - tableName: 'users', - columns: { - id: { type: 'serial' as const, primaryKey: true }, - email: { type: 'varchar' as const, length: 255, nullable: false }, - }, - indexes: { - idx_users_email: { columns: ['email'], unique: false }, - }, - migrations: [], // No migrations provided, will be generated -} as const; - -// SCENARIO 2: SQL-first (only migrations provided) -// Component: tableSchemaComponent({ -// tableName: 'orders', -// migrations: [ -// sqlMigration('001_create', [SQL`CREATE TABLE orders ...`]), -// sqlMigration('002_add_user', [SQL`ALTER TABLE orders ...`]) -// ] -// }) - -// .dumbo/snapshots/schemas/public/orders.snapshot.ts -export const ordersSnapshot = { - tableName: 'orders', - columns: { - // Introspected from database after applying migrations - id: { type: 'serial' as const, primaryKey: false }, - total: { type: 'decimal' as const, precision: 10, scale: 2 }, - user_id: { type: 'int' as const, nullable: true }, - }, - indexes: {}, - migrations: [ - { - name: '001_create', - sqls: ['CREATE TABLE orders (id SERIAL, total DECIMAL(10,2))'], - }, - { - name: '002_add_user', - sqls: ['ALTER TABLE orders ADD COLUMN user_id INT'], - }, - ], -} as const; - -// SCENARIO 3: Hybrid (TypeScript definitions + SQL migrations) -// Component: tableSchemaComponent({ -// tableName: 'posts', -// columns: { id: column('id', ...), title: column('title', ...) }, -// migrations: [sqlMigration('001_custom', [SQL`CREATE TABLE posts ...`])] -// }) - -// .dumbo/snapshots/schemas/public/posts.snapshot.ts -export const postsSnapshot = { - tableName: 'posts', - columns: { - // TypeScript definitions as source of truth - id: { type: 'serial' as const, primaryKey: true }, - title: { type: 'varchar' as const, length: 255, nullable: false }, - }, - indexes: {}, - migrations: [ - { - name: '001_custom', - sqls: [ - 'CREATE TABLE posts (id SERIAL PRIMARY KEY, title VARCHAR(255), internal_field JSONB)', - ], - }, - ], -} as const; - -// .dumbo/snapshots/schemas/public.snapshot.ts -import { usersSnapshot } from './public/users.snapshot'; -import { postsSnapshot } from './public/posts.snapshot'; -import { commentsSnapshot } from './public/comments.snapshot'; - -export const publicSnapshot = { - schemaName: 'public', - tables: { - users: usersSnapshot, - posts: postsSnapshot, - comments: commentsSnapshot, - }, -} as const; - -// .dumbo/snapshots/myapp.snapshot.ts -import { publicSnapshot } from './schemas/public.snapshot'; -import { analyticsSnapshot } from './schemas/analytics.snapshot'; - -export const databaseSnapshot = { - databaseName: 'myapp', - version: '2024-01-15-001', - timestamp: '2024-01-15T10:30:00Z', - schemas: { - public: publicSnapshot, - analytics: analyticsSnapshot, - }, -} as const; -``` - -**Benefits of nested structure**: - -- Minimizes git merge conflicts (each table in separate file) -- Clear ownership and change tracking -- Natural composition mirroring schema hierarchy -- Type-safe imports with TypeScript checking - -**New file to create**: `src/packages/dumbo/src/core/schema/snapshot/schemaSnapshot.ts` - -```typescript -export type ColumnSnapshot = { - type: ColumnDefinition['type']; - nullable?: boolean; - primaryKey?: boolean; - unique?: boolean; - length?: number; - precision?: number; - scale?: number; - defaultValue?: unknown; - references?: { - table: string; - column: string; - onDelete?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; - onUpdate?: 'CASCADE' | 'SET NULL' | 'RESTRICT'; - }; -}; - -export type IndexSnapshot = { - columns: string[]; - unique: boolean; - where?: string; -}; - -export type TableSnapshot = { - tableName: string; - columns: Record; - indexes: Record; - migrations: SQLMigration[]; // Track migrations defined in component -}; - -export type SchemaSnapshot = { - schemaName: string; - tables: Record; - migrations: SQLMigration[]; // Schema-level migrations -}; - -export type DatabaseSnapshot = { - databaseName: string; - version: string; - timestamp: string; - schemas: Record; - migrations: SQLMigration[]; // Database-level migrations -}; -``` - -### Phase 3: Component-Specific Migration Generation - -**IMPORTANT**: Each component is responsible for generating its own migrations when none are provided. - -#### Table Component Migration Generation - -**File to modify**: [src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts) - -```typescript -// Add to tableSchemaComponent.ts -export const generateTableMigrations = ( - table: TableSchemaComponent, - snapshot?: TableSnapshot, -): SQLMigration[] => { - // If migrations already provided, return them (SQL-first mode) - if (table.migrations.length > 0) { - return table.migrations; - } - - // If no columns defined in TypeScript, nothing to generate - if (table.columns.size === 0) { - return []; - } - - const migrations: SQLMigration[] = []; - - if (!snapshot) { - // No snapshot = first time creation - const columnDefinitions = Array.from(table.columns.values()) - .map((col) => generateColumnSQL(col)) - .join(',\n '); - - migrations.push( - sqlMigration(`create_table_${table.tableName}`, [ - SQL`CREATE TABLE IF NOT EXISTS ${table.tableName} ( - ${columnDefinitions} -)`, - ]), - ); - } else { - // Generate ALTER statements based on diff with snapshot - const changes = diffTableWithSnapshot(table, snapshot); - - for (const newColumn of changes.addedColumns) { - migrations.push( - sqlMigration(`add_${newColumn.columnName}_to_${table.tableName}`, [ - SQL`ALTER TABLE ${table.tableName} ADD COLUMN IF NOT EXISTS ${generateColumnSQL(newColumn)}`, - ]), - ); - } - } - - // Indexes are generated as separate migrations - for (const [name, index] of table.indexes) { - if (!snapshot || !snapshot.indexes[name]) { - migrations.push(generateIndexMigration(table.tableName, index)); - } - } - - return migrations; -}; - -// Table component enhanced constructor -export const tableSchemaComponent = ( - options: TableOptions, -): TableSchemaComponent => { - // ... existing implementation ... - - const component = { - // ... existing properties ... - generateMigrations: (snapshot?: TableSnapshot) => - generateTableMigrations(component, snapshot), - }; - - return component; -}; -``` - -#### Index Component Migration Generation - -**File to modify**: [src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/indexSchemaComponent.ts) - -```typescript -export const generateIndexMigration = ( - tableName: string, - index: IndexSchemaComponent, -): SQLMigration => { - const uniqueClause = index.unique ? 'UNIQUE ' : ''; - const columns = index.columns.join(', '); - - return sqlMigration(`create_index_${index.indexName}`, [ - SQL`CREATE ${uniqueClause}INDEX IF NOT EXISTS ${index.indexName} - ON ${tableName}(${columns})`, - ]); -}; -``` - -#### Database Schema Component Migration Generation - -**File to modify**: [src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts) - -```typescript -export const generateSchemaMigrations = ( - schema: DatabaseSchemaSchemaComponent, - snapshot?: SchemaSnapshot, -): SQLMigration[] => { - if (schema.migrations.length > 0) { - return schema.migrations; - } - - const migrations: SQLMigration[] = []; - - // Create schema if doesn't exist - migrations.push( - sqlMigration(`create_schema_${schema.schemaName}`, [ - SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}`, - ]), - ); - - // Tables will generate their own migrations - // But we collect them here for ordering - for (const table of schema.tables.values()) { - migrations.push( - ...table.generateMigrations(snapshot?.tables[table.tableName]), - ); - } - - return migrations; -}; -``` - -### Phase 4: Tree Traversal Migration Collection - -**File to modify**: [src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts](src/packages/dumbo/src/core/schema/components/databaseSchemaComponent.ts) - -The database component is the root and orchestrates migration generation by traversing the tree in the correct order: - -```typescript -export const collectAllMigrations = ( - database: DatabaseSchemaComponent, - snapshot?: DatabaseSnapshot, -): SQLMigration[] => { - const migrations: SQLMigration[] = []; - - // Order matters! Follow this sequence: - // 1. Database-level migrations - // 2. Schema creation - // 3. Table creation (without foreign keys) - // 4. Column additions/modifications - // 5. Indexes - // 6. Foreign keys (future) - // 7. Other constraints (future) - - // If database has explicit migrations, use them - if (database.migrations.length > 0) { - return database.migrations; // SQL-first mode - } - - // 1. Database-level setup - migrations.push( - sqlMigration(`setup_database_${database.databaseName}`, [ - SQL`-- Database setup for ${database.databaseName}`, - ]), - ); - - // 2. Traverse schemas - for (const schema of database.schemas.values()) { - const schemaSnapshot = snapshot?.schemas[schema.schemaName]; - - // Create schema - migrations.push( - sqlMigration(`create_schema_${schema.schemaName}`, [ - SQL`CREATE SCHEMA IF NOT EXISTS ${schema.schemaName}`, - ]), - ); - - // 3. Collect all tables first (structure only, no foreign keys) - const tableCreations: SQLMigration[] = []; - const indexCreations: SQLMigration[] = []; - - for (const table of schema.tables.values()) { - const tableSnapshot = schemaSnapshot?.tables[table.tableName]; - - if (table.migrations.length > 0) { - // Table has explicit migrations, use them - tableCreations.push(...table.migrations); - } else if (table.columns.size > 0) { - // Generate from TypeScript definitions - const tableMigrations = table.generateMigrations(tableSnapshot); - - // Separate table creation from index creation - tableMigrations.forEach((m) => { - if (m.name.includes('index')) { - indexCreations.push(m); - } else { - tableCreations.push(m); - } - }); - } - } - - // Add in correct order - migrations.push(...tableCreations); // All tables first - migrations.push(...indexCreations); // Then all indexes - } - - // Future: Foreign key constraints would go here - // Future: Other constraints, triggers, etc. - - return migrations; -}; - -// Enhanced database component -export const databaseSchemaComponent = ( - options: DatabaseOptions, -): DatabaseSchemaComponent => { - // ... existing implementation ... - - const component = { - // ... existing properties ... - collectAllMigrations: (snapshot?: DatabaseSnapshot) => - collectAllMigrations(component, snapshot), - }; - - return component; -}; -``` - -### Phase 5: Schema Differ - -**New file to create**: `src/packages/dumbo/src/core/schema/differ/schemaDiffer.ts` - -```typescript -export type TableChanges = { - addedColumns: ColumnSchemaComponent[]; - removedColumns: string[]; - modifiedColumns: Array<{ - name: string; - from: ColumnSnapshot; - to: ColumnDefinition; - }>; -}; - -export const diffTableWithSnapshot = ( - table: TableSchemaComponent, - snapshot: TableSnapshot, -): TableChanges => { - const changes: TableChanges = { - addedColumns: [], - removedColumns: [], - modifiedColumns: [], - }; - - // Find added columns - for (const [name, column] of table.columns) { - if (!snapshot.columns[name]) { - changes.addedColumns.push(column); - } - } - - // Find removed columns (might want to warn instead of remove) - for (const name in snapshot.columns) { - if (!table.columns.has(name)) { - changes.removedColumns.push(name); - } - } - - // Find modified columns - for (const [name, column] of table.columns) { - const snapshotColumn = snapshot.columns[name]; - if (snapshotColumn && !columnsEqual(column.definition, snapshotColumn)) { - changes.modifiedColumns.push({ - name, - from: snapshotColumn, - to: column.definition, - }); - } - } - - return changes; -}; - -const columnsEqual = ( - def: ColumnDefinition | undefined, - snap: ColumnSnapshot, -): boolean => { - if (!def) return false; - return ( - def.type === snap.type && - def.nullable === snap.nullable && - def.primaryKey === snap.primaryKey && - def.unique === snap.unique && - def.length === snap.length - ); -}; -``` - -### Phase 6: Type Generation from Schema - -**New file to create**: `src/packages/dumbo/src/core/schema/generators/typeGenerator.ts` - -```typescript -export const generateTypesFromSnapshot = ( - snapshot: DatabaseSnapshot, -): string => { - const lines: string[] = []; - - lines.push('// Auto-generated database types from schema snapshot'); - lines.push('// Do not edit manually - use dumbo schema:generate-types'); - lines.push(''); - - lines.push('export type DatabaseSchema = {'); - - for (const [schemaName, schema] of Object.entries(snapshot.schemas)) { - lines.push(` ${schemaName}: {`); - - for (const [tableName, table] of Object.entries(schema.tables)) { - lines.push(` ${tableName}: {`); - - for (const [columnName, column] of Object.entries(table.columns)) { - const tsType = sqlTypeToTypeScript(column); - lines.push(` ${columnName}: ${tsType};`); - } - - lines.push(' };'); - } - - lines.push(' };'); - } - - lines.push('};'); - - return lines.join('\n'); -}; - -const sqlTypeToTypeScript = (column: ColumnSnapshot): string => { - let baseType: string; - - switch (column.type) { - case 'serial': - case 'int': - case 'bigint': - baseType = 'number'; - break; - case 'varchar': - case 'text': - case 'uuid': - baseType = 'string'; - break; - case 'boolean': - baseType = 'boolean'; - break; - case 'timestamp': - baseType = 'Date'; - break; - case 'json': - case 'jsonb': - baseType = 'unknown'; - break; - default: - baseType = 'unknown'; - } - - return column.nullable ? `${baseType} | null` : baseType; -}; - -// Example generated output: -// export type DatabaseSchema = { -// public: { -// users: { -// id: number; -// email: string; -// name: string | null; -// }; -// posts: { -// id: number; -// userId: number; -// title: string; -// content: string | null; -// }; -// }; -// }; -``` - -### Phase 7: Migration Immutability Validation - -**New file to create**: `src/packages/dumbo/src/core/schema/validators/migrationImmutabilityValidator.ts` - -```typescript -import { SQLMigration } from '../sqlMigration'; - -export type ImmutabilityValidationResult = { - valid: boolean; - error?: string; -}; - -export const validateMigrationImmutability = ( - currentMigrations: ReadonlyArray, - snapshotMigrations: ReadonlyArray, -): ImmutabilityValidationResult => { - // Check that no migrations were removed - for (const snapshotMigration of snapshotMigrations) { - const currentMigration = currentMigrations.find( - (m) => m.name === snapshotMigration.name, - ); - - if (!currentMigration) { - return { - valid: false, - error: - `Migration '${snapshotMigration.name}' was removed from component!\n` + - `Migrations cannot be deleted once defined.\n` + - `If you need to undo a migration, add a new migration that reverses it.`, - }; - } - - // Check that migration content hasn't changed - const snapshotSqls = snapshotMigration.sqls - .map((sql) => sql.toString()) - .join('\n'); - const currentSqls = currentMigration.sqls - .map((sql) => sql.toString()) - .join('\n'); - - if (snapshotSqls !== currentSqls) { - return { - valid: false, - error: - `Migration '${snapshotMigration.name}' has been modified!\n` + - `Original SQL:\n${snapshotSqls}\n\n` + - `Current SQL:\n${currentSqls}\n\n` + - `Migrations must be immutable once defined. Add a new migration instead.`, - }; - } - } - - return { valid: true }; -}; - -// Helper to detect new migrations -export const getNewMigrations = ( - currentMigrations: ReadonlyArray, - snapshotMigrations: ReadonlyArray, -): SQLMigration[] => { - const snapshotNames = new Set(snapshotMigrations.map((m) => m.name)); - return currentMigrations.filter((m) => !snapshotNames.has(m.name)); -}; -``` - -### Phase 8: Schema Validation System - -**New file to create**: `src/packages/dumbo/src/core/schema/validators/migrationValidator.ts` - -```typescript -export type ValidationResult = { - valid: boolean; - errors: Array; - warnings: Array; -}; - -export type ValidationError = { - type: - | 'missing_column' - | 'type_mismatch' - | 'constraint_mismatch' - | 'missing_table'; - message: string; - location: { - component: string; - file?: string; - line?: number; - }; - expected: string; - actual: string; -}; - -export type ValidationWarning = { - type: 'extra_column' | 'extra_index' | 'deprecated_type'; - message: string; -}; - -export const validateMigrationsAgainstSchema = ( - component: SchemaComponent, - options?: { strict: boolean }, -): ValidationResult => { - const result: ValidationResult = { - valid: true, - errors: [], - warnings: [], - }; - - // Only validate if component has both TypeScript definitions AND migrations - if ( - !hasTypeScriptDefinitions(component) || - component.migrations.length === 0 - ) { - return result; - } - - // Simulate migrations in memory to get resulting schema - const resultingSchema = simulateMigrations(component.migrations); - - // Compare with TypeScript definitions - const comparison = compareSchemas(component, resultingSchema, options); - - // Generate detailed error messages - for (const mismatch of comparison.mismatches) { - result.errors.push({ - type: mismatch.type, - message: formatErrorMessage(mismatch), - location: { - component: component.schemaComponentKey, - file: mismatch.file, - line: mismatch.line, - }, - expected: mismatch.expected, - actual: mismatch.actual, - }); - result.valid = false; - } - - return result; -}; - -const formatErrorMessage = (mismatch: SchemaMismatch): string => { - return `Column '${mismatch.column}' mismatch in table '${mismatch.table}': - Expected: ${mismatch.expected} (from ${mismatch.tsFile}:${mismatch.tsLine}) - Actual: ${mismatch.actual} (from ${mismatch.sqlFile}:${mismatch.sqlLine}) - Fix: Either update TypeScript definition or add migration to match`; -}; -``` - -### Phase 9: CLI Commands - -**New CLI commands to implement**: - -```bash -# Compare current TypeScript definitions with snapshot -dumbo schema:diff - -# Generate migrations from diff -dumbo schema:generate [name] -# Example: dumbo schema:generate add-email-to-users - -# Update snapshot to current state -dumbo schema:snapshot - -# Generate TypeScript types from snapshot -dumbo schema:generate-types - -# Validate migrations against TypeScript definitions -dumbo schema:validate - -# Show current schema state -dumbo schema:status - -# Collect and run all migrations from database component -dumbo schema:migrate -``` - -### Phase 10: Migration Orchestrator - -**New file to create**: `src/packages/dumbo/src/core/schema/orchestrator/migrationOrchestrator.ts` - -```typescript -export class MigrationOrchestrator { - constructor( - private database: DatabaseSchemaComponent, - private snapshotPath: string = '.dumbo/snapshots', - private migrationsPath: string = '.dumbo/migrations', - ) {} - - async processMigrations(): Promise { - // Load latest snapshot if exists - const snapshot = await this.loadSnapshot(); - - // Validate migration immutability - const validation = await this.validateMigrations(snapshot); - if (!validation.valid) { - throw new Error(validation.error); - } - - // Process user-provided migrations - const userProvidedMigrations = - await this.processUserProvidedMigrations(snapshot); - - // Generate migrations for TypeScript-defined components - const generatedMigrations = - await this.generateMigrationsFromDefinitions(snapshot); - - return { - userProvided: userProvidedMigrations, - generated: generatedMigrations, - all: [...userProvidedMigrations, ...generatedMigrations], - }; - } - - private async processUserProvidedMigrations( - snapshot: DatabaseSnapshot | null, - ): Promise { - const newMigrations: SQLMigration[] = []; - - // Process database-level migrations - if (this.database.migrations.length > 0) { - const snapshotMigrations = snapshot?.migrations || []; - const newDbMigrations = getNewMigrations( - this.database.migrations, - snapshotMigrations, - ); - - for (const migration of newDbMigrations) { - // Write to migrations folder - await this.writeMigrationToFile( - migration, - 'database', - this.database.schemaComponentKey, - ); - newMigrations.push(migration); - } - } - - // Traverse and process schema and table migrations - for (const schema of this.database.schemas.values()) { - const schemaSnapshot = snapshot?.schemas[schema.schemaName]; - - // Process schema migrations - if (schema.migrations.length > 0) { - const snapshotMigrations = schemaSnapshot?.migrations || []; - const newSchemaMigrations = getNewMigrations( - schema.migrations, - snapshotMigrations, - ); - - for (const migration of newSchemaMigrations) { - await this.writeMigrationToFile( - migration, - 'schema', - schema.schemaComponentKey, - ); - newMigrations.push(migration); - } - } - - // Process table migrations - for (const table of schema.tables.values()) { - const tableSnapshot = schemaSnapshot?.tables[table.tableName]; - - if (table.migrations.length > 0) { - const snapshotMigrations = tableSnapshot?.migrations || []; - const newTableMigrations = getNewMigrations( - table.migrations, - snapshotMigrations, - ); - - for (const migration of newTableMigrations) { - await this.writeMigrationToFile( - migration, - 'table', - table.schemaComponentKey, - ); - newMigrations.push(migration); - } - } - } - } - - return newMigrations; - } - - private async writeMigrationToFile( - migration: SQLMigration, - type: 'database' | 'schema' | 'table', - componentKey: string, - ): Promise { - const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filename = `${timestamp}-${migration.name}.sql`; - - const header = [ - `-- Source: provided (from component migrations array)`, - `-- Component: ${componentKey}`, - `-- Type: ${type}`, - `-- Migration name: ${migration.name}`, - `-- Written at: ${new Date().toISOString()}`, - '', - '', - ].join('\n'); - - const content = - header + migration.sqls.map((sql) => sql.toString()).join(';\n') + ';'; - - await writeFile(`${this.migrationsPath}/${filename}`, content); - } - - private async validateMigrations( - snapshot: DatabaseSnapshot | null, - ): Promise { - if (!snapshot) return { valid: true }; - - // Validate database migrations - const dbValidation = validateMigrationImmutability( - this.database.migrations, - snapshot.migrations || [], - ); - if (!dbValidation.valid) return dbValidation; - - // Validate schema and table migrations - for (const schema of this.database.schemas.values()) { - const schemaSnapshot = snapshot.schemas[schema.schemaName]; - if (!schemaSnapshot) continue; - - const schemaValidation = validateMigrationImmutability( - schema.migrations, - schemaSnapshot.migrations || [], - ); - if (!schemaValidation.valid) return schemaValidation; - - for (const table of schema.tables.values()) { - const tableSnapshot = schemaSnapshot.tables[table.tableName]; - if (!tableSnapshot) continue; - - const tableValidation = validateMigrationImmutability( - table.migrations, - tableSnapshot.migrations || [], - ); - if (!tableValidation.valid) return tableValidation; - } - } - - return { valid: true }; - } - - async runMigrations(dumbo: Dumbo): Promise { - const processed = await this.processMigrations(); - - // Group migrations by type for correct ordering - const grouped = this.groupMigrations(processed.all); - - // Run in correct order - await this.runMigrationGroup(dumbo, grouped.schemas); - await this.runMigrationGroup(dumbo, grouped.tables); - await this.runMigrationGroup(dumbo, grouped.columns); - await this.runMigrationGroup(dumbo, grouped.indexes); - // Future: grouped.relationships, grouped.constraints - - // Update snapshot after successful migration - await this.updateSnapshot(); - } - - private async loadSnapshot(): Promise { - try { - // Load root snapshot which imports all nested snapshots - const module = await import( - `${this.snapshotPath}/${this.database.databaseName}.snapshot.ts` - ); - return module.databaseSnapshot; - } catch (error) { - // No snapshot found, will generate everything from scratch - return null; - } - } - - private async updateSnapshot(): Promise { - // Generate new snapshots for each component - await this.generateDatabaseSnapshot(this.database); - } - - private async generateDatabaseSnapshot( - database: DatabaseSchemaComponent, - ): Promise { - // Generate nested snapshot files - for (const schema of database.schemas.values()) { - await this.generateSchemaSnapshot(schema); - } - - // Generate root database snapshot that imports schemas - const imports = Array.from(database.schemas.values()) - .map( - (s) => - `import { ${s.schemaName}Snapshot } from './schemas/${s.schemaName}.snapshot';`, - ) - .join('\n'); - - const migrationsStr = this.formatMigrationsForSnapshot(database.migrations); - - const content = `${imports} - -export const databaseSnapshot = { - databaseName: '${database.databaseName}', - version: '${new Date().toISOString().split('T')[0]}-001', - timestamp: '${new Date().toISOString()}', - schemas: { - ${Array.from(database.schemas.values()) - .map((s) => `${s.schemaName}: ${s.schemaName}Snapshot`) - .join(',\n ')} - }, - migrations: ${migrationsStr} -} as const;`; - - await writeFile( - `${this.snapshotPath}/${database.databaseName}.snapshot.ts`, - content, - ); - } - - private async generateSchemaSnapshot( - schema: DatabaseSchemaSchemaComponent, - ): Promise { - // Generate table snapshots first - for (const table of schema.tables.values()) { - await this.generateTableSnapshot(schema.schemaName, table); - } - - // Generate schema snapshot that imports tables - const imports = Array.from(schema.tables.values()) - .map( - (t) => - `import { ${t.tableName}Snapshot } from './${schema.schemaName}/${t.tableName}.snapshot';`, - ) - .join('\n'); - - const migrationsStr = this.formatMigrationsForSnapshot(schema.migrations); - - const content = `${imports} - -export const ${schema.schemaName}Snapshot = { - schemaName: '${schema.schemaName}', - tables: { - ${Array.from(schema.tables.values()) - .map((t) => `${t.tableName}: ${t.tableName}Snapshot`) - .join(',\n ')} - }, - migrations: ${migrationsStr} -} as const;`; - - await writeFile( - `${this.snapshotPath}/schemas/${schema.schemaName}.snapshot.ts`, - content, - ); - } - - private async generateTableSnapshot( - schemaName: string, - table: TableSchemaComponent, - ): Promise { - const migrationsStr = this.formatMigrationsForSnapshot(table.migrations); - - const content = `export const ${table.tableName}Snapshot = { - tableName: '${table.tableName}', - columns: { - ${Array.from(table.columns.entries()) - .map(([name, col]) => this.generateColumnSnapshot(name, col)) - .join(',\n ')} - }, - indexes: { - ${Array.from(table.indexes.entries()) - .map(([name, idx]) => this.generateIndexSnapshot(name, idx)) - .join(',\n ')} - }, - migrations: ${migrationsStr} -} as const;`; - - await writeFile( - `${this.snapshotPath}/schemas/${schemaName}/${table.tableName}.snapshot.ts`, - content, - ); - } - - private formatMigrationsForSnapshot( - migrations: ReadonlyArray, - ): string { - if (migrations.length === 0) return '[]'; - - const migrationStrs = migrations.map((m) => { - const sqlsStr = m.sqls - .map((sql) => `'${sql.toString().replace(/'/g, "\\'")}'`) - .join(', '); - return `{ name: '${m.name}', sqls: [${sqlsStr}] }`; - }); - - return `[\n ${migrationStrs.join(',\n ')}\n ]`; - } - - private groupMigrations(migrations: SQLMigration[]) { - return { - schemas: migrations.filter((m) => m.name.includes('schema')), - tables: migrations.filter((m) => m.name.includes('create_table')), - columns: migrations.filter( - (m) => m.name.includes('add_') || m.name.includes('alter_'), - ), - indexes: migrations.filter((m) => m.name.includes('index')), - }; - } -} -``` - -## Testing Strategy - -### Unit Tests - -Following the pattern in [src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts](src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts): - -```typescript -// src/packages/dumbo/src/core/schema/generators/migrationGenerator.unit.spec.ts -import { describe, it, assert } from 'vitest'; -import { tableSchemaComponent } from '../components/tableSchemaComponent'; -import { column } from '../components/columnSchemaComponent'; - -describe('generateTableMigrations', () => { - it('should generate CREATE TABLE from TypeScript definition', () => { - const table = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { - type: 'varchar', - length: 255, - nullable: false, - }), - }, - }); - - const migrations = table.generateMigrations(); - - assert.equal(migrations.length, 1); - assert.match(migrations[0].sqls[0], /CREATE TABLE IF NOT EXISTS users/); - assert.match(migrations[0].sqls[0], /id SERIAL PRIMARY KEY/); - assert.match(migrations[0].sqls[0], /email VARCHAR\(255\) NOT NULL/); - }); - - it('should detect added columns when comparing with snapshot', () => { - const snapshot = { - tableName: 'users', - columns: { - id: { type: 'serial' as const, primaryKey: true }, - }, - indexes: {}, - }; - - const table = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }), - }, - }); - - const migrations = table.generateMigrations(snapshot); - - assert.equal(migrations.length, 1); - assert.match( - migrations[0].sqls[0], - /ALTER TABLE users ADD COLUMN IF NOT EXISTS email/, - ); - }); -}); -``` - -### Integration Tests - -```typescript -// src/packages/dumbo/src/core/schema/orchestrator/migrationOrchestrator.integration.spec.ts -import { describe, it, assert, beforeEach } from 'vitest'; -import { MigrationOrchestrator } from './migrationOrchestrator'; -import { createTestDatabase } from '../../test/utils'; - -describe('MigrationOrchestrator integration', () => { - let db: Dumbo; - - beforeEach(async () => { - db = await createTestDatabase(); - }); - - it('should apply schema changes in correct order', async () => { - const schema = databaseSchemaComponent({ - databaseName: 'test', - schemas: { - public: databaseSchemaSchemaComponent({ - schemaName: 'public', - tables: { - users: tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }), - }, - }), - posts: tableSchemaComponent({ - tableName: 'posts', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - userId: column('userId', { - type: 'int', - references: { table: 'users', column: 'id' }, - }), - }, - }), - }, - }), - }, - }); - - const orchestrator = new MigrationOrchestrator(schema); - await orchestrator.runMigrations(db); - - // Verify tables were created - const tables = await db.query(` - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - `); - - assert.deepEqual(tables.map((t) => t.table_name).sort(), [ - 'posts', - 'users', - ]); - - // Verify columns exist - const columns = await db.query(` - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = 'users' - `); - - assert.equal(columns.length, 2); - }); -}); -``` - -### End-to-End Tests - -```typescript -// src/packages/dumbo/src/core/schema/e2e/schemaEvolution.e2e.spec.ts -describe('Schema evolution E2E', () => { - it('should handle complete schema lifecycle', async () => { - // 1. Initial schema definition - const v1Schema = createSchema({ version: 1 }); - await orchestrator.runMigrations(v1Schema); - await orchestrator.updateSnapshot(); - - // 2. Schema modification - const v2Schema = createSchema({ - version: 2, - addColumn: { table: 'users', column: 'phone' }, - }); - - // 3. Diff detection - const diff = await orchestrator.diff(v2Schema); - assert.equal(diff.changes.length, 1); - assert.equal(diff.changes[0].type, 'add_column'); - - // 4. Migration generation - const migrations = await orchestrator.generateMigrations(); - assert.equal(migrations.length, 1); - - // 5. Migration application - await orchestrator.runMigrations(v2Schema); - - // 6. Snapshot update - await orchestrator.updateSnapshot(); - - // 7. Type generation - const types = await generateTypesFromSnapshot(); - assert.match(types, /phone: string | null/); - - // 8. Validation - const validation = await validateMigrationsAgainstSchema(v2Schema); - assert.equal(validation.valid, true); - }); -}); -``` - -## Complete Flow Example - -This section demonstrates how all pieces work together in practice: - -### Step 1: Initial Component Definition with SQL Migration - -```typescript -// User defines component with custom SQL migration -const usersTable = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }), - }, - migrations: [ - sqlMigration('001_custom_create', [ - SQL`CREATE TABLE users ( - id SERIAL PRIMARY KEY, - email VARCHAR(255) UNIQUE, - internal_tracking JSONB DEFAULT '{}' - )`, - ]), - ], -}); -``` - -### Step 2: First Orchestrator Run - -```typescript -const orchestrator = new MigrationOrchestrator(database); - -// Process migrations (validates, diffs, writes new ones) -await orchestrator.processMigrations(); -// - No snapshot exists, so no validation needed -// - Detects '001_custom_create' as new -// - Writes to: .dumbo/migrations/2024-01-15-001-custom_create.sql -``` - -### Step 3: Generated Files After First Run - -```sql --- .dumbo/migrations/2024-01-15-001-custom_create.sql --- Source: provided (from component migrations array) --- Component: sc:dumbo:table:users --- Type: table --- Migration name: 001_custom_create --- Written at: 2024-01-15T10:30:00Z - -CREATE TABLE users ( - id SERIAL PRIMARY KEY, - email VARCHAR(255) UNIQUE, - internal_tracking JSONB DEFAULT '{}' -); -``` - -```typescript -// .dumbo/snapshots/schemas/public/users.snapshot.ts -export const usersSnapshot = { - tableName: 'users', - columns: { - id: { type: 'serial' as const, primaryKey: true }, - email: { type: 'varchar' as const, length: 255, nullable: false }, - }, - indexes: {}, - migrations: [ - { - name: '001_custom_create', - sqls: [ - "CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}')", - ], - }, - ], -} as const; -``` - -### Step 4: User Adds New Column and Migration - -```typescript -// User updates component -const usersTable = tableSchemaComponent({ - tableName: 'users', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - email: column('email', { type: 'varchar', length: 255 }), - name: column('name', { type: 'varchar', length: 100 }), // NEW! - }, - migrations: [ - sqlMigration('001_custom_create', [ - SQL`CREATE TABLE users ( - id SERIAL PRIMARY KEY, - email VARCHAR(255) UNIQUE, - internal_tracking JSONB DEFAULT '{}' - )`, - ]), - sqlMigration('002_add_name', [ - // NEW! - SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)`, - ]), - ], -}); -``` - -### Step 5: Second Orchestrator Run - -```typescript -await orchestrator.processMigrations(); -// 1. Validates '001_custom_create' hasn't changed ✓ -// 2. Detects '002_add_name' as new -// 3. Writes new migration to file -// 4. NO generated migration for 'name' column (user provided SQL) -``` - -### Step 6: Error Case - Modified Migration - -```typescript -// User accidentally modifies existing migration (BAD!) -const usersTable = tableSchemaComponent({ - tableName: 'users', - migrations: [ - sqlMigration('001_custom_create', [ - SQL`CREATE TABLE users ( - id BIGSERIAL PRIMARY KEY, // Changed from SERIAL! - email VARCHAR(255) UNIQUE, - internal_tracking JSONB DEFAULT '{}' - )`, - ]), - sqlMigration('002_add_name', [ - SQL`ALTER TABLE users ADD COLUMN name VARCHAR(100)`, - ]), - ], -}); - -await orchestrator.processMigrations(); -// Throws error: -// Migration '001_custom_create' has been modified! -// Original SQL: -// CREATE TABLE users (id SERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}') -// -// Current SQL: -// CREATE TABLE users (id BIGSERIAL PRIMARY KEY, email VARCHAR(255) UNIQUE, internal_tracking JSONB DEFAULT '{}') -// -// Migrations must be immutable once defined. Add a new migration instead. -``` - -### Step 7: Mixed Mode - TypeScript + SQL - -```typescript -// Another table with TypeScript-first approach -const postsTable = tableSchemaComponent({ - tableName: 'posts', - columns: { - id: column('id', { type: 'serial', primaryKey: true }), - title: column('title', { type: 'varchar', length: 255 }), - userId: column('userId', { - type: 'int', - references: { table: 'users', column: 'id' }, - }), - }, - // No migrations provided - will be generated! -}); - -await orchestrator.processMigrations(); -// Generates and writes: .dumbo/migrations/2024-01-15-002-create_table_posts.sql -// Content: -// -- Source: generated (from TypeScript definitions) -// -- Component: sc:dumbo:table:posts -// -- Generated at: 2024-01-15T10:45:00Z -// CREATE TABLE IF NOT EXISTS posts ( -// id SERIAL PRIMARY KEY, -// title VARCHAR(255) NOT NULL, -// userId INT REFERENCES users(id) -// ); -``` - -### Step 8: Type Generation - -```typescript -// Generate types from snapshots -const types = await generateTypesFromSnapshot(databaseSnapshot); - -// Results in: -export type DatabaseSchema = { - public: { - users: { - id: number; - email: string; - name: string | null; - }; - posts: { - id: number; - title: string; - userId: number; - }; - }; -}; - -// Can be used in Pongo query builder: -const result = await pongo - .from('users') - .where('email', '=', 'test@example.com') - .select(['id', 'name']); -// TypeScript knows result is { id: number; name: string | null }[] -``` - -## Migration Strategy for Existing Code - -1. **Backward Compatible**: Existing SQL-first code continues to work unchanged -2. **Progressive Enhancement**: Can add TypeScript definitions gradually -3. **Opt-in Validation**: Validation only runs when explicitly enabled -4. **Safe Defaults**: Use `CREATE IF NOT EXISTS` when no snapshot found - -## Success Criteria - -1. ✅ Each component can generate its own migrations -2. ✅ Database component traverses tree in correct order -3. ✅ Can use raw SQL migrations without TypeScript (backward compatible) -4. ✅ Can validate SQL migrations against TypeScript definitions -5. ✅ Nested snapshot system minimizes merge conflicts -6. ✅ Types can be generated from schema for Pongo integration -7. ✅ CLI provides clear workflow for schema evolution -8. ✅ Type safety maintained throughout -9. ✅ No breaking changes to existing API - -## Key Architectural Decisions - -1. **Each component generates its own migrations** - Encapsulation and single responsibility -2. **Database component orchestrates collection** - Single entry point for migration generation -3. **Tree traversal follows dependency order** - Schemas → Tables → Columns → Indexes → Foreign Keys -4. **TypeScript definitions are source of truth when present** - Override SQL for intended state -5. **Nested snapshot structure** - Each table/schema in separate file to minimize conflicts -6. **TypeScript snapshot format** - Type safety and IDE support over JSON simplicity -7. **Snapshots track component migrations** - Enable detection of new user-provided migrations -8. **All migrations go to `.dumbo/migrations/`** - Both generated and user-provided, uniform handling -9. **Migration immutability enforced** - Once in snapshot, migrations cannot be modified -10. **CREATE IF NOT EXISTS as default** - Safe behavior when no snapshot exists -11. **Indexes generate as separate migrations** - Clean separation of concerns -12. **Validation is opt-in** - No breaking changes, teams choose their strictness level -13. **Type generation from snapshots** - Enable strongly-typed Pongo query builders - -## Implementation Order - -1. **Phase 1**: Column definitions with SQL generation -2. **Phase 2**: Nested snapshot system structure with migration tracking -3. **Phase 3**: Component-specific migration generation (each component file) -4. **Phase 4**: Tree traversal in database component -5. **Phase 5**: Schema differ for change detection -6. **Phase 6**: Type generation from schema snapshots -7. **Phase 7**: Migration immutability validation -8. **Phase 8**: Schema validation system with detailed errors -9. **Phase 9**: CLI commands for workflow -10. **Phase 10**: Migration orchestrator with diffing and validation - -Each phase builds on the previous one and can be tested independently. The key insight is that each component knows how to generate its own SQL, and the database component knows how to traverse the tree in the correct order to collect all migrations. - -## Future Enhancements (Out of Scope) - -1. **Query Builder Integration**: Use generated types for strongly-typed Pongo queries -2. **Schema Introspection**: Read current schema from database and generate TypeScript -3. **Migration Rollback**: Track down migrations and generate reversal scripts -4. **Multi-Database Support**: PostgreSQL, MySQL, SQLite-specific features -5. **Partial Schema Definitions**: Allow defining subset of columns for gradual adoption -6. **Smart Merge Tool**: CLI command to help resolve snapshot conflicts -7. **Performance Optimization**: Cache component trees for large schemas - -## Summary - -This plan unifies SQL migrations and TypeScript schema definitions through: - -### Core Innovations - -1. **Dual-purpose snapshots** - Track both current schema state AND component migrations -2. **Migration diffing** - Detect new user-provided migrations by comparing with snapshots -3. **Immutability enforcement** - Prevent dangerous modification of existing migrations -4. **Unified migration storage** - All migrations (generated/provided) go to `.dumbo/migrations/` -5. **Nested snapshot structure** - Minimize git conflicts by separating tables into files - -### Key Benefits - -- **Flexibility** - Support SQL-first, TypeScript-first, or hybrid approaches -- **Safety** - Migration immutability prevents accidental schema corruption -- **Type Safety** - Generate types from snapshots for Pongo query builders -- **Gradual Adoption** - Add TypeScript definitions progressively to SQL-first projects -- **Clear Audit Trail** - Track all migrations with source metadata - -### Migration Workflow - -1. Component defines schema (TypeScript) and/or migrations (SQL) -2. Orchestrator validates immutability against snapshot -3. New user-provided migrations are written to files -4. TypeScript definitions generate missing migrations -5. Snapshots update to reflect current state + migrations -6. Types are generated for strongly-typed queries - -This architecture provides a robust foundation for schema evolution while maintaining backwards compatibility and enabling progressive enhancement. - -## References - -- Component hierarchy uses URN system with `sc:dumbo:database:{name}` pattern -- Migration bubbling happens at lines 79-83 in schemaComponent.ts -- Recent refactoring in commit 03fb40a changed to use `Exclude` instead of `Omit` -- Test patterns follow existing dumboSchema.unit.spec.ts structure -- Integration target is Pongo schema system in src/packages/pongo/src/core/schema/index.ts diff --git a/src/packages/dumbo/src/core/schema/REVIEW.MD b/src/packages/dumbo/src/core/schema/REVIEW.MD deleted file mode 100644 index 771dda1b..00000000 --- a/src/packages/dumbo/src/core/schema/REVIEW.MD +++ /dev/null @@ -1,201 +0,0 @@ -Deep Analysis of the Architecture -After thoroughly rereading the plan, here's my multi-perspective analysis: -From a Regular User's Perspective (Storage Library Consumer) -Strengths: -Progressive Disclosure of Complexity - Users can start simple (SQL-only or TypeScript-only) and gradually adopt more features. This is excellent API design. -Familiar Patterns - The API resembles popular ORMs (Prisma, TypeORM) making adoption easier. -Clear Error Messages - The immutability validation provides actionable feedback rather than cryptic errors. -Type Safety Benefits - The generated types for query builders provide immediate value that developers can feel in their IDE. -Potential Pain Points: -Hidden Complexity in Snapshots - The dual-purpose snapshots (schema state + migration tracking) might be confusing initially. Users might not understand why snapshots contain their migrations. -Directory Proliferation - The .dumbo/ folder with nested snapshots could feel overwhelming for small projects. -Mental Model Shift - Understanding when TypeScript is truth vs when SQL is truth requires careful documentation and good examples. -Migration Immutability - While safer, this might frustrate users coming from Rails/Django where you can edit migrations before deploying. -From a Software Architect's Perspective -Architectural Strengths: -Separation of Concerns - Each component generating its own SQL is textbook single responsibility principle. This makes the system highly testable and maintainable. -Command-Query Separation - The clear distinction between: -Commands (processMigrations, runMigrations) -Queries (loadSnapshot, validateMigrations) -Pure functions (generateMigrations, diffTableWithSnapshot) -Temporal Decoupling - Snapshots act as a temporal boundary, allowing the system to reason about "then vs now" without querying the database. -Composability - The tree structure with proper traversal order handles complex dependencies elegantly. -Architectural Concerns: -Snapshot as Dual Source of Truth - Using snapshots for BOTH schema state AND migration tracking violates single responsibility. Consider splitting into: -Schema snapshots (current state for diffing) -Migration manifest (tracking what's been processed) -File System as Database - The .dumbo/migrations/ directory becomes a critical data store. What happens with: -Concurrent writes? -File system permissions? -CI/CD environments? -Type Generation Timing - Generating types from snapshots rather than from the database means types might not reflect reality if migrations fail partially. -Validation vs Enforcement - The system validates but doesn't enforce schema consistency. A team member could still manually edit the database. -Architecture as Code Analysis -Elegant Design Patterns: -Builder Pattern in dumboSchema API: -dumboSchema.database('myapp', { -public: dumboSchema.schema('public', { -users: dumboSchema.table('users', {...}) -}) -}) -Template Method Pattern in migration generation - each component implements its own generateMigrations() but follows the same structure. -Visitor Pattern in tree traversal - the orchestrator visits each component in dependency order. -Memento Pattern - Snapshots act as mementos capturing component state at a point in time. -Code Smells to Address: -Large Parameter Lists - The orchestrator constructor and several functions have many parameters that could be grouped into configuration objects. -String Manipulation for SQL - Building SQL through string concatenation (even with template literals) is risky. Consider a SQL builder abstraction. -Implicit Ordering Dependencies - The migration grouping relies on naming conventions (includes('schema'), includes('create_table')). This is fragile. -Mixed Abstraction Levels - The orchestrator handles both high-level workflow and low-level file I/O. Consider extracting a MigrationWriter service. -Philosophical Considerations -The Snapshot Paradox: The snapshots serve as both historical record (what migrations were defined) and current state (what the schema looks like). This dual nature is philosophically interesting but practically complex. It's like Git storing both diffs AND the current state in the same structure. Migration Immutability Trade-off: Enforcing immutability is philosophically sound (history should be append-only) but practically challenging. Real-world schema evolution is messy, and sometimes you need to fix a broken migration. TypeScript as Truth: The decision to make TypeScript the source of truth when present is bold. It prioritizes developer experience over database reality, which could lead to drift. -Recommendations for Improvement -Split Snapshot Responsibilities: -Keep schema snapshots for state -Add a separate .dumbo/manifest.json for tracking processed migrations -Add Schema Verification: -After running migrations, query information_schema to verify the result matches expectations -Generate types from actual database state, not snapshots -Implement Conflict Resolution: -Add a dumbo schema:merge command for snapshot conflicts -Use three-way merge logic (base, mine, theirs) -Consider Event Sourcing: -Treat migrations as events in an event store -Snapshots become projections of those events -Enables time-travel debugging -Add Dry-Run Capability: -Let users preview what migrations would be generated/applied -Essential for production deployments -Overall Assessment -This is a sophisticated and well-thought-out architecture that successfully bridges the gap between SQL-first and TypeScript-first workflows. The use of snapshots for migration tracking is innovative, though it adds complexity. The system would benefit from: -Clearer separation between state tracking and migration manifest -Runtime verification that generated types match database reality -Better tooling for snapshot conflict resolution -More explicit handling of edge cases (partial migration failure, concurrent modifications) -The architecture shows maturity in its backwards compatibility and progressive enhancement approach. It's clearly designed by someone who understands both the technical challenges and the human factors of schema management. Grade: B+ Strong foundation with room for refinement in the snapshot dual-responsibility and runtime verification aspects. - -######### - -What if snapshots were stored as jsons would that help in understanding the split? -Excellent question! Let me think through how JSON snapshots would affect the architecture, particularly around the split of responsibilities. -JSON Snapshots: Impact Analysis -How JSON Would Help with the Split -Current Issue with TypeScript Snapshots: -// .dumbo/snapshots/schemas/public/users.snapshot.ts -export const usersSnapshot = { -tableName: 'users', -columns: { /_ schema state _/ }, -migrations: [ /* migration tracking */ ] // Mixed responsibility! -} as const; -With JSON + Split Responsibilities: -// .dumbo/snapshots/schemas/public/users.schema.json -{ -"tableName": "users", -"columns": { -"id": { "type": "serial", "primaryKey": true }, -"email": { "type": "varchar", "length": 255 } -}, -"indexes": {} -} - -// .dumbo/manifests/schemas/public/users.manifest.json -{ -"component": "sc:dumbo:table:users", -"processedMigrations": [ -{ -"name": "001_custom_create", -"hash": "sha256:abcd1234...", -"processedAt": "2024-01-15T10:30:00Z", -"source": "provided" -} -] -} -Benefits of JSON Format -Clear Separation of Concerns -_.schema.json = Current state (for diffing) -_.manifest.json = Migration history (for tracking) -No temptation to mix responsibilities in a single file -Better Git Diffs -{ -"columns": { -"id": { "type": "serial" }, - -- "email": { "type": "varchar", "length": 255 } - } - } - Much cleaner than TypeScript object diffs with imports and type annotations. - Language Agnostic - Other tools (Python scripts, CI/CD, database admin tools) can read/write - No TypeScript compilation needed to access snapshot data - Simpler Merge Resolution - JSON merge tools are mature and widespread - Can use jq or similar for command-line manipulation - Easier to write custom merge strategies - Drawbacks of JSON - Loss of Type Safety - No compile-time checking of snapshot structure - Need runtime validation (though you need this anyway) - No Imports/Composition - Can't use TypeScript's import system for nested structure - Would need to manually resolve file references - Verbosity for Complex Structures - No as const assertions - More verbose without TypeScript's type inference - Hybrid Approach: Best of Both Worlds - What if we used JSON for storage but TypeScript for types? - // .dumbo/snapshots/types.ts (generated) - import usersSchema from './schemas/public/users.schema.json'; - import postsSchema from './schemas/public/posts.schema.json'; - -export type DatabaseSchema = { -public: { -users: typeof usersSchema; -posts: typeof postsSchema; -} -}; -Recommended Structure with JSON -.dumbo/ -schemas/ # Current state (JSON) -database.json -public/ -schema.json -tables/ -users.json -posts.json -manifests/ # Migration tracking (JSON) -database.json -public/ -schema.json -tables/ -users.json -posts.json -migrations/ # SQL files -2024-01-15-001-create.sql -generated/ # Generated TypeScript -types.ts # Generated from schemas/ -manifest-types.ts # Types for manifest structure -Impact on the Architecture -Cleaner Responsibilities: -Schema JSON: Pure state representation -Manifest JSON: Pure migration history -Generated TS: Type safety without mixing concerns -Better Tooling Opportunities: -dumbo schema:diff - Compare JSON files -dumbo manifest:verify - Check manifest against database -dumbo schema:validate - JSON schema validation -Simpler Mental Model: -"Schemas describe what IS" -"Manifests describe what WAS DONE" -"Migrations describe HOW TO DO IT" -My Recommendation -Yes, JSON would significantly help with understanding and implementing the split. Specifically: -Use JSON for snapshots (both schema and manifest) -Keep them in separate directories -Generate TypeScript types from the JSON for type safety -Store the actual SQL in migrations as a third, distinct concern -This gives you: -Clear separation of concerns -Better git diffs and merge resolution -Language-agnostic data files -Type safety through code generation -Simpler mental model -The only real loss is the elegance of TypeScript imports for composition, but that's a worthwhile trade-off for the clarity gained. Would you like me to revise the plan to use this JSON-based approach with properly separated concerns? diff --git a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts index 1f7ca340..af9d3ba2 100644 --- a/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/columnSchemaComponent.ts @@ -6,18 +6,23 @@ import { } from '../schemaComponent'; export type ColumnURNType = 'sc:dumbo:column'; -export type ColumnURN = `${ColumnURNType}:${string}`; +export type ColumnURN = + `${ColumnURNType}:${ColumnName}`; export const ColumnURNType: ColumnURNType = 'sc:dumbo:column'; -export const ColumnURN = ({ name }: { name: string }): ColumnURN => - `${ColumnURNType}:${name}`; +export const ColumnURN = ({ + name, +}: { + name: ColumnName; +}): ColumnURN => `${ColumnURNType}:${name}`; export type ColumnSchemaComponent< ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, + ColumnName extends string = string, > = SchemaComponent< - ColumnURN, + ColumnURN, Readonly<{ - columnName: string; + columnName: ColumnName; }> > & SQLColumnToken; @@ -31,14 +36,17 @@ export type ColumnSchemaComponentOptions< SchemaComponentOptions; export const columnSchemaComponent = < - ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, - TOptions extends + const ColumnType extends AnyColumnTypeToken | string = + | AnyColumnTypeToken + | string, + const TOptions extends ColumnSchemaComponentOptions = ColumnSchemaComponentOptions, + const ColumnName extends string = string, >( params: { - columnName: string; + columnName: ColumnName; } & TOptions, -): ColumnSchemaComponent & +): ColumnSchemaComponent & (TOptions extends { notNull: true } | { primaryKey: true } ? { notNull: true } : { notNull?: false }) => { @@ -66,7 +74,7 @@ export const columnSchemaComponent = < type, }; - return result as ColumnSchemaComponent & + return result as ColumnSchemaComponent & (TOptions extends { notNull: true } | { primaryKey: true } ? { notNull: true } : { notNull?: false }); diff --git a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts index a01dbeb5..99074006 100644 --- a/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/databaseSchemaSchemaComponent.ts @@ -12,15 +12,16 @@ import { } from './tableSchemaComponent'; export type DatabaseSchemaURNType = 'sc:dumbo:database_schema'; -export type DatabaseSchemaURN = `${DatabaseSchemaURNType}:${string}`; +export type DatabaseSchemaURN = + `${DatabaseSchemaURNType}:${SchemaName}`; export const DatabaseSchemaURNType: DatabaseSchemaURNType = 'sc:dumbo:database_schema'; -export const DatabaseSchemaURN = ({ +export const DatabaseSchemaURN = ({ name, }: { - name: string; -}): DatabaseSchemaURN => `${DatabaseSchemaURNType}:${name}`; + name: SchemaName; +}): DatabaseSchemaURN => `${DatabaseSchemaURNType}:${name}`; export type DatabaseSchemaTables< Tables extends AnyTableSchemaComponent = AnyTableSchemaComponent, @@ -28,10 +29,11 @@ export type DatabaseSchemaTables< export type DatabaseSchemaSchemaComponent< Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + SchemaName extends string = string, > = SchemaComponent< - DatabaseSchemaURN, + DatabaseSchemaURN, Readonly<{ - schemaName: string; + schemaName: SchemaName; tables: ReadonlyMap & Tables; addTable: (table: string | TableSchemaComponent) => TableSchemaComponent; }> @@ -39,18 +41,22 @@ export type DatabaseSchemaSchemaComponent< export type AnyDatabaseSchemaSchemaComponent = // eslint-disable-next-line @typescript-eslint/no-explicit-any - DatabaseSchemaSchemaComponent; + DatabaseSchemaSchemaComponent; export const databaseSchemaSchemaComponent = < - Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const SchemaName extends string = string, >({ schemaName, tables, ...migrationsOrComponents }: { - schemaName: string; + schemaName: SchemaName; tables?: Tables; -} & SchemaComponentOptions): DatabaseSchemaSchemaComponent => { +} & SchemaComponentOptions): DatabaseSchemaSchemaComponent< + Tables, + SchemaName +> => { const base = schemaComponent(DatabaseSchemaURN({ name: schemaName }), { migrations: migrationsOrComponents.migrations ?? [], components: [ diff --git a/src/packages/dumbo/src/core/schema/components/relationships/collectReferencesErrors.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/collectReferencesErrors.type.spec.ts new file mode 100644 index 00000000..f470ed68 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/collectReferencesErrors.type.spec.ts @@ -0,0 +1,272 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect } from '../../../testing'; +import { dumboSchema } from '../../dumboSchema'; +import type { SchemaColumnName } from './relationshipTypes'; +import type { + CollectReferencesErrors, + ColumnReferenceExistanceError, + ColumnReferenceTypeMismatchError, +} from './relationshipValidation'; + +const { column, table, schema } = dumboSchema; +const { BigInteger, Varchar, Integer } = SQL.column.type; + +void describe('CollectReferencesErrors', () => { + const usersTable = table('users', { + columns: { + id: column('id', BigInteger), + name: column('name', Varchar('max')), + age: column('age', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + title: column('title', Varchar('max')), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + }); + + type TestSchemas = { + public: typeof _publicSchema; + }; + + void it('returns empty array when all references are valid', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [SchemaColumnName<'public', 'users', 'id'>]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type _Then = Expect>; + }); + + void it('returns empty array for multiple valid references', () => { + type Columns = readonly [ + SchemaColumnName<'public', 'posts', 'user_id'>, + SchemaColumnName<'public', 'posts', 'title'>, + ]; + type References = readonly [ + SchemaColumnName<'public', 'users', 'id'>, + SchemaColumnName<'public', 'users', 'name'>, + ]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type _Then = Expect>; + }); + + void it('collects error for missing schema', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [SchemaColumnName<'nonexistent', 'users', 'id'>]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceExistanceError<'missing_schema', 'nonexistent.users.id'>, + ]; + + type _Then = Expect>; + }); + + void it('collects error for missing table', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [ + SchemaColumnName<'public', 'nonexistent', 'id'>, + ]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceExistanceError<'missing_table', 'public.nonexistent.id'>, + ]; + + type _Then = Expect>; + }); + + void it('collects error for missing column', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [ + SchemaColumnName<'public', 'users', 'nonexistent'>, + ]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceExistanceError< + 'missing_column', + 'public.users.nonexistent' + >, + ]; + + type _Then = Expect>; + }); + + void it('collects error for type mismatch', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [SchemaColumnName<'public', 'users', 'name'>]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceTypeMismatchError< + 'public.users.name', + 'VARCHAR', + 'BIGINT' + >, + ]; + + type _Then = Expect>; + }); + + void it('collects multiple errors for different invalid references', () => { + type Columns = readonly [ + SchemaColumnName<'public', 'posts', 'user_id'>, + SchemaColumnName<'public', 'posts', 'title'>, + ]; + type References = readonly [ + SchemaColumnName<'public', 'users', 'nonexistent'>, + SchemaColumnName<'public', 'users', 'age'>, + ]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceExistanceError< + 'missing_column', + 'public.users.nonexistent' + >, + ColumnReferenceTypeMismatchError< + 'public.users.age', + 'INTEGER', + 'VARCHAR' + >, + ]; + + type _Then = Expect>; + }); + + void it('collects only errors, skipping valid references', () => { + type Columns = readonly [ + SchemaColumnName<'public', 'posts', 'user_id'>, + SchemaColumnName<'public', 'posts', 'title'>, + SchemaColumnName<'public', 'posts', 'id'>, + ]; + type References = readonly [ + SchemaColumnName<'public', 'users', 'id'>, + SchemaColumnName<'public', 'users', 'age'>, + SchemaColumnName<'public', 'users', 'id'>, + ]; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type Expected = [ + ColumnReferenceTypeMismatchError< + 'public.users.age', + 'INTEGER', + 'VARCHAR' + >, + ]; + + type _Then = Expect>; + }); + + void it('returns empty array for empty input tuples', () => { + type Columns = readonly []; + type References = readonly []; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas + >; + + type _Then = Expect>; + }); + + void it('accumulates errors with pre-existing errors', () => { + type Columns = readonly [SchemaColumnName<'public', 'posts', 'user_id'>]; + type References = readonly [ + SchemaColumnName<'public', 'users', 'nonexistent'>, + ]; + type ExistingError = ColumnReferenceExistanceError< + 'missing_column', + 'public.foo.bar' + >; + + type Result = CollectReferencesErrors< + Columns, + References, + 'public', + 'posts', + TestSchemas, + [ExistingError] + >; + + type Expected = [ + ExistingError, + ColumnReferenceExistanceError< + 'missing_column', + 'public.users.nonexistent' + >, + ]; + + type _Then = Expect>; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/collectRelationshipErrors.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/collectRelationshipErrors.type.spec.ts new file mode 100644 index 00000000..7b687582 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/collectRelationshipErrors.type.spec.ts @@ -0,0 +1,346 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import type { InferTableSchemaComponentColumns } from '../tableSchemaComponent'; +import { relationship } from './relationshipTypes'; +import type { CollectRelationshipErrors } from './relationshipValidation'; + +const { schema, table, column } = dumboSchema; +const { Integer, BigInteger, Varchar } = SQL.column.type; + +void describe('CollectRelationshipErrors', () => { + const usersTable = table('users', { + columns: { + id: column('id', BigInteger), + tenant_id: column('tenant_id', Integer), + name: column('name', Varchar('max')), + }, + }); + + const tenantsTable = table('tenants', { + columns: { + id: column('id', Integer), + name: column('name', Varchar('max')), + }, + }); + + const postsTable = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + tenant_id: column('tenant_id', Integer), + title: column('title', Varchar('max')), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'many-to-one'), + tenant: relationship(['tenant_id'], ['public.tenants.id'], 'many-to-one'), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + tenants: tenantsTable, + }); + + type TestSchemas = { + public: typeof _publicSchema; + }; + + type PostsTable = typeof postsTable; + type PostsColumns = InferTableSchemaComponentColumns; + + void it('returns empty array when all relationships are valid', () => { + type Result = CollectRelationshipErrors< + PostsColumns, + PostsTable['relationships'], + PostsTable, + typeof _publicSchema, + TestSchemas + >; + + type _Then = Expect>; + }); + + void it('collects errors for invalid references', () => { + const _postsTableWithBadRef = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + }, + relationships: { + user: relationship( + ['user_id'], + ['public.users.nonexistent'], + 'many-to-one', + ), + }, + }); + + type BadPostsTable = typeof _postsTableWithBadRef; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + BadPostsTable['relationships'], + BadPostsTable, + typeof _publicSchema, + TestSchemas + >; + + type Expected = [ + TypeValidationResult< + false, + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_column'; + reference: 'public.users.nonexistent'; + }, + ]; + } + >, + ]; + + type _Then = Expect>; + }); + + void it('collects errors from multiple invalid relationships', () => { + const _postsTableMultipleErrors = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + tenant_id: column('tenant_id', Integer), + }, + relationships: { + user: relationship( + ['user_id'], + ['nonexistent.users.id'], + 'many-to-one', + ), + tenant: relationship( + ['tenant_id'], + ['public.missing.id'], + 'many-to-one', + ), + }, + }); + + type MultiErrorTable = typeof _postsTableMultipleErrors; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + MultiErrorTable['relationships'], + MultiErrorTable, + typeof _publicSchema, + TestSchemas + >; + + type Expected = [ + TypeValidationResult< + false, + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + } + >, + TypeValidationResult< + false, + { + relationship: 'tenant'; + errors: [ + { + errorCode: 'missing_table'; + reference: 'public.missing.id'; + }, + ]; + } + >, + ]; + + type _Then = Expect>; + }); + + void it('collects type mismatch errors', () => { + const _postsTableTypeMismatch = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + }, + relationships: { + user: relationship(['user_id'], ['public.users.name'], 'many-to-one'), + }, + }); + + type MismatchTable = typeof _postsTableTypeMismatch; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + MismatchTable['relationships'], + MismatchTable, + typeof _publicSchema, + TestSchemas + >; + + type Expected = [ + TypeValidationResult< + false, + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.name'; + referenceType: 'VARCHAR'; + columnTypeName: 'BIGINT'; + }, + ]; + } + >, + ]; + + type _Then = Expect>; + }); + + void it('collects length mismatch errors', () => { + const _postsTableLengthMismatch = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + tenant_id: column('tenant_id', Integer), + }, + relationships: { + composite: relationship( + ['user_id', 'tenant_id'], + ['public.users.id'], + 'many-to-one', + ), + }, + }); + + type LengthMismatchTable = typeof _postsTableLengthMismatch; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + LengthMismatchTable['relationships'], + LengthMismatchTable, + typeof _publicSchema, + TestSchemas + >; + + type Expected = [ + TypeValidationResult< + false, + { + relationship: 'composite'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: readonly ['user_id', 'tenant_id']; + references: readonly ['public.users.id']; + }, + ]; + } + >, + ]; + + type _Then = Expect>; + }); + + void it('skips valid relationships and only collects errors', () => { + const _postsTableMixed = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + tenant_id: column('tenant_id', Integer), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'many-to-one'), + tenant: relationship( + ['tenant_id'], + ['public.tenants.bad'], + 'many-to-one', + ), + }, + }); + + type MixedTable = typeof _postsTableMixed; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + MixedTable['relationships'], + MixedTable, + typeof _publicSchema, + TestSchemas + >; + + type Expected = [ + TypeValidationResult< + false, + { + relationship: 'tenant'; + errors: [ + { + errorCode: 'missing_column'; + reference: 'public.tenants.bad'; + }, + ]; + } + >, + ]; + + type _Then = Expect>; + }); + + void it('handles composite foreign keys', () => { + const compositeUsersTable = table('users', { + columns: { + id: column('id', BigInteger), + tenant_id: column('tenant_id', Integer), + }, + }); + + const compositePostsTable = table('posts', { + columns: { + id: column('id', BigInteger), + user_id: column('user_id', BigInteger), + tenant_id: column('tenant_id', Integer), + }, + relationships: { + userTenant: relationship( + ['user_id', 'tenant_id'], + ['public.users.id', 'public.users.tenant_id'], + 'many-to-one', + ), + }, + }); + + const _compositeSchema = schema('public', { + users: compositeUsersTable, + posts: compositePostsTable, + }); + + type CompositeSchemas = { + public: typeof _compositeSchema; + }; + + type CompositePostsTable = typeof compositePostsTable; + + type Result = CollectRelationshipErrors< + InferTableSchemaComponentColumns, + CompositePostsTable['relationships'], + CompositePostsTable, + typeof _compositeSchema, + CompositeSchemas + >; + + type _Then = Expect>; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.ts b/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.ts new file mode 100644 index 00000000..6a28eac5 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.ts @@ -0,0 +1,155 @@ +export type Join< + T extends readonly string[], + Sep extends string, +> = T extends readonly [ + infer First extends string, + ...infer Rest extends readonly string[], +] + ? Rest extends readonly [] + ? First + : `${First}${Sep}${Join}` + : ''; + +export type IndentErrors = + Messages extends readonly [ + infer First extends string, + ...infer Rest extends readonly string[], + ] + ? [` - ${First}`, ...IndentErrors] + : []; + +type ExtractSchemaFromReference = + T extends `${infer Schema}.${string}.${string}` ? Schema : never; + +type ExtractTableFromReference = + T extends `${string}.${infer Table}.${string}` ? Table : never; + +type ExtractColumnFromReference = + T extends `${string}.${string}.${infer Column}` ? Column : never; + +type TupleLength = T extends { length: infer L } + ? L + : never; + +export type FormatSingleError = E extends { + errorCode: 'reference_columns_mismatch'; + invalidColumns: infer InvalidCols extends readonly string[]; + availableColumns: infer AvailableCols extends readonly string[]; +} + ? `Invalid columns: ${Join}. Available columns: ${Join}` + : E extends { + errorCode: 'reference_length_mismatch'; + columns: infer Cols extends readonly string[]; + references: infer Refs extends readonly string[]; + } + ? `Column count mismatch: ${TupleLength} columns ([${Join}]) but ${TupleLength} references ([${Join}])` + : E extends { + errorCode: 'missing_schema'; + reference: infer Ref extends string; + } + ? `Schema "${ExtractSchemaFromReference}" does not exist (${Ref})` + : E extends { + errorCode: 'missing_table'; + reference: infer Ref extends string; + } + ? `Table "${ExtractTableFromReference}" does not exist in schema "${ExtractSchemaFromReference}" (${Ref})` + : E extends { + errorCode: 'missing_column'; + reference: infer Ref extends string; + } + ? `Column "${ExtractColumnFromReference}" does not exist in table "${ExtractSchemaFromReference}.${ExtractTableFromReference}" (${Ref})` + : E extends { + errorCode: 'type_mismatch'; + reference: infer Ref extends string; + referenceType: infer RefType extends string; + columnTypeName: infer ColType extends string; + } + ? `Type mismatch: column type "${ColType}" does not match referenced column type "${RefType}" (${Ref})` + : never; + +type FormatErrorMessages = + Errors extends readonly [ + infer First, + ...infer Rest extends readonly unknown[], + ] + ? [FormatSingleError, ...FormatErrorMessages] + : []; + +export type FormatRelationshipBlock = E extends { + relationship: infer RelName extends string; + errors: infer Errors extends readonly unknown[]; +} + ? Join< + [ + `Relationship "${RelName}":`, + ...IndentErrors>, + ], + '\n' + > + : never; + +type IndentLine = ` ${Line}`; + +type IndentRelationshipBlock = + Block extends `${infer FirstLine}\n${infer Rest}` + ? `${IndentLine}\n${IndentRelationshipBlock}` + : IndentLine; + +type FormatRelationshipBlocks = + RelErrors extends readonly [ + infer First, + ...infer Rest extends readonly unknown[], + ] + ? Rest extends readonly [] + ? IndentRelationshipBlock> + : `${IndentRelationshipBlock>}\n${FormatRelationshipBlocks}` + : ''; + +export type FormatTableLevel = E extends { + table: infer TableName extends string; + errors: infer RelErrors extends readonly unknown[]; +} + ? `Table "${TableName}":\n${FormatRelationshipBlocks}` + : never; + +type IndentTableBlock = + Block extends `${infer FirstLine}\n${infer Rest}` + ? ` ${FirstLine}\n${IndentTableBlock}` + : ` ${Block}`; + +type FormatTableBlocks = + TableErrors extends readonly [ + infer First, + ...infer Rest extends readonly unknown[], + ] + ? Rest extends readonly [] + ? IndentTableBlock> + : `${IndentTableBlock>}\n${FormatTableBlocks}` + : ''; + +export type FormatSchemaLevel = E extends { + schema: infer SchemaName extends string; + errors: infer TableErrors extends readonly unknown[]; +} + ? `Schema "${SchemaName}":\n${FormatTableBlocks}` + : never; + +type FormatSchemaBlocks = + SchemaErrors extends readonly [ + infer First, + ...infer Rest extends readonly unknown[], + ] + ? Rest extends readonly [] + ? FormatSchemaLevel + : `${FormatSchemaLevel}\n${FormatSchemaBlocks}` + : ''; + +export type FormatDatabaseValidationErrors = + FormatSchemaBlocks; + +export type FormatValidationErrors = Result extends { + valid: false; + error: infer Errors extends readonly unknown[]; +} + ? `Relationship validation errors:\n\n${FormatDatabaseValidationErrors}` + : never; diff --git a/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.type.spec.ts new file mode 100644 index 00000000..32c8332b --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/formatRelationshipErrors.type.spec.ts @@ -0,0 +1,404 @@ +import { describe, it } from 'node:test'; +import type { Equals, Expect } from '../../../testing'; +import type { + Join, + IndentErrors, + FormatSingleError, + FormatRelationshipBlock, + FormatTableLevel, + FormatSchemaLevel, + FormatDatabaseValidationErrors, + FormatValidationErrors, +} from './formatRelationshipErrors'; + +void describe('Join', () => { + void it('concatenates empty array to empty string', () => { + type Result = Join<[], ', '>; + type _Then = Expect>; + }); + + void it('handles single element', () => { + type Result = Join<['foo'], ', '>; + type _Then = Expect>; + }); + + void it('concatenates multiple elements with separator', () => { + type Result = Join<['foo', 'bar', 'baz'], ', '>; + type _Then = Expect>; + }); + + void it('handles different separators', () => { + type Result = Join<['a', 'b', 'c'], ' | '>; + type _Then = Expect>; + }); +}); + +void describe('IndentErrors', () => { + void it('formats empty array', () => { + type Result = IndentErrors<[]>; + type _Then = Expect>; + }); + + void it('adds bullet and indent to single message', () => { + type Result = IndentErrors<['Missing column']>; + type _Then = Expect>; + }); + + void it('adds bullet and indent to multiple messages', () => { + type Result = IndentErrors<['First error', 'Second error', 'Third error']>; + type Expected = [' - First error', ' - Second error', ' - Third error']; + type _Then = Expect>; + }); +}); + +void describe('FormatSingleError', () => { + void it('formats reference_columns_mismatch error', () => { + type Error = { + errorCode: 'reference_columns_mismatch'; + invalidColumns: ['col1', 'col2']; + availableColumns: ['id', 'name']; + }; + type Result = FormatSingleError; + type Expected = 'Invalid columns: col1, col2. Available columns: id, name'; + type _Then = Expect>; + }); + + void it('formats reference_length_mismatch error', () => { + type Error = { + errorCode: 'reference_length_mismatch'; + columns: ['col1', 'col2']; + references: ['public.users.id']; + }; + type Result = FormatSingleError; + type Expected = + 'Column count mismatch: 2 columns ([col1, col2]) but 1 references ([public.users.id])'; + type _Then = Expect>; + }); + + void it('formats missing_schema error', () => { + type Error = { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }; + type Result = FormatSingleError; + type Expected = + 'Schema "nonexistent" does not exist (nonexistent.users.id)'; + type _Then = Expect>; + }); + + void it('formats missing_table error', () => { + type Error = { + errorCode: 'missing_table'; + reference: 'public.nonexistent.id'; + }; + type Result = FormatSingleError; + type Expected = + 'Table "nonexistent" does not exist in schema "public" (public.nonexistent.id)'; + type _Then = Expect>; + }); + + void it('formats missing_column error', () => { + type Error = { + errorCode: 'missing_column'; + reference: 'public.users.nonexistent'; + }; + type Result = FormatSingleError; + type Expected = + 'Column "nonexistent" does not exist in table "public.users" (public.users.nonexistent)'; + type _Then = Expect>; + }); + + void it('formats type_mismatch error', () => { + type Error = { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }; + type Result = FormatSingleError; + type Expected = + 'Type mismatch: column type "BIGINT" does not match referenced column type "INTEGER" (public.users.id)'; + type _Then = Expect>; + }); +}); + +void describe('FormatRelationshipBlock', () => { + void it('formats single error in relationship', () => { + type Input = { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + }; + type Result = FormatRelationshipBlock; + type Expected = `Relationship "user": + - Schema "nonexistent" does not exist (nonexistent.users.id)`; + type _Then = Expect>; + }); + + void it('formats multiple errors in relationship', () => { + type Input = { + relationship: 'posts'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: ['col1', 'col2']; + references: ['public.users.id']; + }, + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }; + type Result = FormatRelationshipBlock; + type Expected = `Relationship "posts": + - Column count mismatch: 2 columns ([col1, col2]) but 1 references ([public.users.id]) + - Type mismatch: column type "BIGINT" does not match referenced column type "INTEGER" (public.users.id)`; + type _Then = Expect>; + }); +}); + +void describe('FormatTableLevel', () => { + void it('formats single relationship error in table', () => { + type Input = { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + }, + ]; + }; + type Result = FormatTableLevel; + type Expected = `Table "posts": + Relationship "user": + - Schema "nonexistent" does not exist (nonexistent.users.id)`; + type _Then = Expect>; + }); + + void it('formats multiple relationship errors in table', () => { + type Input = { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + { + relationship: 'category'; + errors: [ + { + errorCode: 'missing_table'; + reference: 'public.categories.id'; + }, + ]; + }, + ]; + }; + type Result = FormatTableLevel; + type Expected = `Table "posts": + Relationship "user": + - Type mismatch: column type "BIGINT" does not match referenced column type "INTEGER" (public.users.id) + Relationship "category": + - Table "categories" does not exist in schema "public" (public.categories.id)`; + type _Then = Expect>; + }); +}); + +void describe('FormatSchemaLevel', () => { + void it('formats single table error in schema', () => { + type Input = { + schema: 'public'; + errors: [ + { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + }, + ]; + }, + ]; + }; + type Result = FormatSchemaLevel; + type Expected = `Schema "public": + Table "posts": + Relationship "user": + - Schema "nonexistent" does not exist (nonexistent.users.id)`; + type _Then = Expect>; + }); + + void it('formats multiple table errors in schema', () => { + type Input = { + schema: 'public'; + errors: [ + { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + ]; + }, + { + table: 'comments'; + errors: [ + { + relationship: 'post'; + errors: [ + { + errorCode: 'missing_column'; + reference: 'public.posts.post_id'; + }, + ]; + }, + ]; + }, + ]; + }; + type Result = FormatSchemaLevel; + type Expected = `Schema "public": + Table "posts": + Relationship "user": + - Type mismatch: column type "BIGINT" does not match referenced column type "INTEGER" (public.users.id) + Table "comments": + Relationship "post": + - Column "post_id" does not exist in table "public.posts" (public.posts.post_id)`; + type _Then = Expect>; + }); +}); + +void describe('FormatDatabaseValidationErrors', () => { + void it('formats database validation errors with multiple schemas', () => { + type Input = [ + { + schema: 'public'; + errors: [ + { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + ]; + }, + ]; + }, + { + schema: 'auth'; + errors: [ + { + table: 'sessions'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_table'; + reference: 'auth.users.id'; + }, + ]; + }, + ]; + }, + ]; + }, + ]; + type Result = FormatDatabaseValidationErrors; + type Expected = `Schema "public": + Table "posts": + Relationship "user": + - Type mismatch: column type "BIGINT" does not match referenced column type "INTEGER" (public.users.id) +Schema "auth": + Table "sessions": + Relationship "user": + - Table "users" does not exist in schema "auth" (auth.users.id)`; + type _Then = Expect>; + }); +}); + +void describe('FormatValidationErrors', () => { + void it('formats validation result with errors', () => { + type Input = { + valid: false; + error: [ + { + schema: 'public'; + errors: [ + { + table: 'posts'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + }, + ]; + }, + ]; + }, + ]; + }; + type Result = FormatValidationErrors; + type Expected = `Relationship validation errors: + +Schema "public": + Table "posts": + Relationship "user": + - Schema "nonexistent" does not exist (nonexistent.users.id)`; + type _Then = Expect>; + }); + + void it('returns never for valid result', () => { + type Input = { + valid: true; + error: undefined; + }; + type Result = FormatValidationErrors; + type _Then = Expect>; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/index.ts b/src/packages/dumbo/src/core/schema/components/relationships/index.ts index e29b349e..1451ea17 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/index.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/index.ts @@ -1,3 +1,5 @@ export * from './relationshipTypes'; export * from './relationshipValidation'; + +export * from './formatRelationshipErrors'; diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts index 731402d9..5d38be3a 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipTypes.ts @@ -8,18 +8,21 @@ import type { TableColumnNames, TableColumns, TableSchemaComponent, + Writable, } from '..'; +import type { ColumnTypeToken } from '../../../sql/tokens/columnTokens'; +import type { NotEmptyTuple } from '../../../typing'; export type ExtractSchemaNames = DB extends DatabaseSchemaComponent - ? keyof Schemas & string + ? keyof Schemas : never; export type ExtractTableNames = Schema extends DatabaseSchemaSchemaComponent< infer Tables extends DatabaseSchemaTables > - ? keyof Tables & string + ? keyof Tables : never; export type ExtractColumnNames
= @@ -27,24 +30,180 @@ export type ExtractColumnNames
= ? TableColumnNames> : never; -export type AllColumnReferences = - DB extends DatabaseSchemaComponent +export type ExtractColumnTypeName = + T extends ColumnTypeToken< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any, + infer TypeName, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any + > + ? Uppercase + : never; + +export type AllColumnTypes = { + [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< + infer Tables + > + ? Writable<{ + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns + > + ? Writable<{ + [ColumnName in keyof Columns]: { + columnTypeName: ExtractColumnTypeName< + Columns[ColumnName]['type'] + >; + }; + }> + : never; + }> + : never; +}; + +export type AllColumnReferences = { + [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< + infer Tables + > + ? { + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns + > + ? { + [ColumnName in keyof Columns]: `${SchemaName & + string}.${TableName & string}.${ColumnName & string}`; + }[keyof Columns] + : never; + }[keyof Tables] + : never; +}[keyof Schemas]; + +export type AllColumnTypesInSchema< + Schema extends AnyDatabaseSchemaSchemaComponent, +> = + Schema extends DatabaseSchemaSchemaComponent + ? { + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns + > + ? { + [ColumnName in keyof Columns]: { + columnTypeName: ExtractColumnTypeName< + Columns[ColumnName]['type'] + >; + }; + } + : never; + } + : never; + +export type AllColumnReferencesInSchema< + Schema extends AnyDatabaseSchemaSchemaComponent, + SchemaName extends string, +> = + Schema extends DatabaseSchemaSchemaComponent ? { - [SchemaName in keyof Schemas]: Schemas[SchemaName] extends DatabaseSchemaSchemaComponent< - infer Tables + [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< + infer Columns > ? { - [TableName in keyof Tables]: Tables[TableName] extends TableSchemaComponent< - infer Columns - > - ? { - [ColumnName in keyof Columns]: `${SchemaName & - string}.${TableName & string}.${ColumnName & string}`; - }[keyof Columns] - : never; - }[keyof Tables] + [ColumnName in keyof Columns]: `${SchemaName & string}.${TableName & + string}.${ColumnName & string}`; + }[keyof Columns] : never; - }[keyof Schemas] + }[keyof Tables] + : never; + +export type NormalizeReference< + Path extends string, + CurrentSchema extends string, + CurrentTable extends string, +> = Path extends `${infer Schema}.${infer Table}.${infer Column}` + ? `${Schema}.${Table}.${Column}` + : Path extends `${infer Table}.${infer Column}` + ? `${CurrentSchema}.${Table}.${Column}` + : Path extends string + ? `${CurrentSchema}.${CurrentTable}.${Path}` + : never; + +export type NormalizeColumnPath< + References extends readonly string[], + SchemaName extends string, + TableName extends string, +> = References extends readonly [infer First, ...infer Rest] + ? First extends string + ? Rest extends readonly string[] + ? readonly [ + NormalizeReference, + ...NormalizeColumnPath, + ] + : readonly [] + : readonly [] + : readonly []; + +export type ColumnName = `${ColName}`; + +export type TableColumnName< + TableName extends string = string, + ColName extends string = string, +> = `${TableName}.${ColName}`; + +export type SchemaColumnName< + SchemaName extends string = string, + TableName extends string = string, + ColumnName extends string = string, +> = `${SchemaName}.${TableName}.${ColumnName}`; + +export type ColumnPath< + SchemaName extends string = string, + TableName extends string = string, + ColName extends string = string, +> = + | SchemaColumnName + | TableColumnName + | ColumnName; + +export type ColumnReference< + SchemaName extends string = string, + TableName extends string = string, + ColumnName extends string = string, +> = { schemaName: SchemaName; tableName: TableName; columnName: ColumnName }; + +export type ColumnPathToReference< + Reference extends ColumnPath = ColumnPath, + CurrentSchema extends string = string, + CurrentTable extends string = string, +> = + NormalizeReference< + Reference, + CurrentSchema, + CurrentTable + > extends `${infer S}.${infer T}.${infer C}` + ? { schemaName: S; tableName: T; columnName: C } + : never; + +export type ParseReferencePath = + Path extends `${infer Schema}.${infer Table}.${infer Column}` + ? { schema: Schema; table: Table; column: Column } + : never; + +export type LookupColumnType = + ParseReferencePath extends { + schema: infer S; + table: infer T; + column: infer C; + } + ? S extends keyof AllTypes + ? T extends keyof AllTypes[S] + ? C extends keyof AllTypes[S][T] + ? AllTypes[S][T][C] extends { columnTypeName: infer TypeName } + ? TypeName + : never + : never + : never + : never : never; export type RelationshipType = @@ -53,24 +212,48 @@ export type RelationshipType = | 'many-to-one' | 'many-to-many'; -export type RelationshipDefinition = { - readonly columns: readonly Columns[]; - readonly references: readonly References[]; - readonly type: RelationshipType; +export type RelationshipDefinition< + Columns extends string = string, + Reference extends string = string, + RelType extends RelationshipType = RelationshipType, +> = { + readonly columns: NotEmptyTuple; + readonly references: NotEmptyTuple; + readonly type: RelType; }; +export type AnyTableRelationshipDefinition = RelationshipDefinition< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any +>; + +export type AnyTableRelationshipDefinitionWithColumns< + Columns extends string = string, +> = RelationshipDefinition< + Columns, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + any +>; + export type TableRelationships = Record< string, - RelationshipDefinition + AnyTableRelationshipDefinitionWithColumns >; export const relationship = < const Columns extends readonly string[], const References extends readonly string[], + const RelType extends RelationshipType = RelationshipType, >( columns: Columns, references: References, - type: RelationshipType, + type: RelType, ) => { return { columns, @@ -79,4 +262,4 @@ export const relationship = < } as const; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyRelationshipDefinition = RelationshipDefinition; +export type AnyRelationshipDefinition = RelationshipDefinition; diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts index c37a3ed0..0eb6c31d 100644 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts +++ b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.ts @@ -1,188 +1,531 @@ import type { - AnyDatabaseSchemaComponent, + AnyColumnSchemaComponent, AnyDatabaseSchemaSchemaComponent, - DatabaseSchemaComponent, + ColumnSchemaComponent, + DatabaseSchemas, DatabaseSchemaSchemaComponent, } from '..'; +import type { FormatValidationErrors } from './formatRelationshipErrors'; +import type { AnyColumnTypeToken, ColumnTypeToken } from '../../../sql'; +import type { + ALL, + AND, + AnyTypeValidationError, + AnyTypeValidationFailed, + FailOnFirstTypeValidationError, + FilterNotExistingInUnion, + HaveTuplesTheSameLength, + IF, + IsEmptyTuple, + IsNotEmptyTuple, + KeysOfString, + MapRecordCollectErrors, + NotEmptyTuple, + TypeValidationError, + TypeValidationResult, + TypeValidationSuccess, + UnwrapTypeValidationErrors, + ZipTuplesCollectErrors, +} from '../../../typing'; import type { AnyTableSchemaComponent, + TableColumns, TableSchemaComponent, } from '../tableSchemaComponent'; -import type { TableColumnNames } from '../tableTypesInference'; import type { - AllColumnReferences, - AnyRelationshipDefinition, + AnyTableRelationshipDefinition, + AnyTableRelationshipDefinitionWithColumns, + NormalizeColumnPath, + SchemaColumnName, + TableRelationships, } from './relationshipTypes'; -export type ValidationResult< - Valid extends boolean, - Error = never, -> = Valid extends true ? { valid: true } : { valid: false; error: Error }; +export type RelationshipColumnsMismatchError< + ColumnPath extends SchemaColumnName = SchemaColumnName, +> = { + valid: false; + error: { + errorCode: 'reference_columns_mismatch'; + invalidColumns: ColumnPath[]; + availableColumns: ColumnPath[]; + }; +}; + +export type RelationshipReferencesLengthMismatchError< + ColumnPath extends SchemaColumnName = SchemaColumnName, +> = { + valid: false; + error: { + errorCode: 'reference_length_mismatch'; + columns: ColumnPath[]; + references: ColumnPath[]; + }; +}; -type GetArrayLength = T['length']; +export type ColumnReferenceExistanceError< + ErrorCode extends 'missing_schema' | 'missing_table' | 'missing_column' = + | 'missing_schema' + | 'missing_table' + | 'missing_column', + ColumnPath extends SchemaColumnName = SchemaColumnName, +> = { + valid: false; + error: { + errorCode: ErrorCode; + reference: ColumnPath; + }; +}; + +export type ColumnReferenceTypeMismatchError< + Reference extends SchemaColumnName = SchemaColumnName, + ReferenceTypeName extends string = string, + ColumnTypeName extends string = string, +> = { + valid: false; + error: { + errorCode: 'type_mismatch'; + reference: Reference; + referenceType: ReferenceTypeName; + columnTypeName: ColumnTypeName; + }; +}; + +export type NoError = TypeValidationSuccess; + +export type ColumnReferenceError = + | ColumnReferenceExistanceError + | ColumnReferenceTypeMismatchError; + +export type RelationshipValidationError = + | RelationshipColumnsMismatchError + | RelationshipReferencesLengthMismatchError + | ColumnReferenceError; export type ValidateRelationshipLength< - FK extends { columns: readonly unknown[]; references: readonly unknown[] }, + Rel extends AnyTableRelationshipDefinition, +> = IF< + ALL< + [ + HaveTuplesTheSameLength, + IsNotEmptyTuple, + IsNotEmptyTuple, + ] + >, + TypeValidationSuccess, + TypeValidationResult< + false, + { + errorCode: 'reference_length_mismatch'; + columns: Rel['columns']; + references: Rel['references']; + } + > +>; + +export type ValidateRelationshipColumns< + Relationship extends AnyTableRelationshipDefinition, + ValidColumns extends TableColumns, > = - GetArrayLength extends GetArrayLength - ? ValidationResult - : ValidationResult< - false, - `Foreign key columns and references must have the same length. Got ${GetArrayLength} columns and ${GetArrayLength} references.` - >; + FilterNotExistingInUnion< + Relationship['columns'], + KeysOfString + > extends infer InvalidColumns extends NotEmptyTuple + ? IF< + AND< + IsEmptyTuple, + IsNotEmptyTuple + >, + TypeValidationSuccess, + TypeValidationResult< + false, + { + errorCode: 'reference_columns_mismatch'; + invalidColumns: InvalidColumns; + availableColumns: KeysOfString; + } + > + > + : TypeValidationSuccess; -type FindInvalidColumns< - Columns extends readonly string[], - ValidColumns extends string, - Invalid extends string[] = [], -> = Columns extends readonly [infer First, ...infer Rest] - ? First extends string - ? Rest extends readonly string[] - ? First extends ValidColumns - ? FindInvalidColumns - : FindInvalidColumns - : Invalid - : Invalid - : Invalid; - -type AllInTuple< - Tuple extends readonly string[], - Union extends string, -> = Tuple extends readonly [infer First, ...infer Rest] - ? First extends Union - ? Rest extends readonly string[] - ? AllInTuple - : true - : false - : true; +export type ValidateColumnReference< + ColReference extends SchemaColumnName, + Schemas extends DatabaseSchemas, +> = + ColReference extends SchemaColumnName< + infer SchemaName, + infer TableName, + infer ColumnName + > + ? SchemaName extends keyof Schemas + ? TableName extends keyof Schemas[SchemaName]['tables'] + ? Schemas[SchemaName]['tables'][TableName] extends TableSchemaComponent< + infer Columns, + infer _TableName, + infer _Relationships + > + ? ColumnName extends keyof Columns + ? Columns[ColumnName] + : ColumnReferenceExistanceError< + 'missing_column', + `${SchemaName}.${TableName}.${ColumnName}` + > + : never + : ColumnReferenceExistanceError< + 'missing_table', + `${SchemaName}.${TableName}.${ColumnName}` + > + : ColumnReferenceExistanceError< + 'missing_schema', + `${SchemaName}.${TableName}.${ColumnName}` + > + : never; -export type ValidateRelationshipColumns< - FK extends { columns: readonly string[] }, - ValidColumns extends string, +export type ValidateColumnTypeMatch< + RefColumnType extends AnyColumnTypeToken | string = + | AnyColumnTypeToken + | string, + ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, + Reference extends SchemaColumnName = SchemaColumnName, > = - AllInTuple extends true - ? ValidationResult - : ValidationResult< - false, - `Invalid foreign key columns: ${FindInvalidColumns extends infer Invalid ? (Invalid extends string[] ? Invalid[number] : never) : never}. Available columns: ${ValidColumns}` - >; + ColumnType extends ColumnTypeToken< + infer _JsType, + infer ColumnTypeName, + infer _TProps + > + ? RefColumnType extends ColumnTypeToken< + infer _JsType, + infer RefColumnTypeName, + infer _TProps + > + ? RefColumnTypeName extends ColumnTypeName + ? TypeValidationSuccess + : ColumnReferenceTypeMismatchError< + Reference, + RefColumnTypeName, + ColumnTypeName + > + : RefColumnType extends ColumnTypeName + ? TypeValidationSuccess + : ColumnReferenceTypeMismatchError< + Reference, + Extract, + ColumnTypeName + > + : RefColumnType extends ColumnTypeToken< + infer _JsType, + infer RefColumnTypeName, + infer _TProps + > + ? RefColumnTypeName extends ColumnType + ? TypeValidationSuccess + : ColumnReferenceTypeMismatchError< + Reference, + RefColumnTypeName, + Extract + > + : RefColumnType extends ColumnType + ? TypeValidationSuccess + : ColumnReferenceTypeMismatchError< + Reference, + Extract, + Extract + >; -type FindInvalidReferences< - References extends readonly string[], - ValidReferences extends string, - Invalid extends string[] = [], -> = References extends readonly [infer First, ...infer Rest] - ? First extends string - ? Rest extends readonly string[] - ? First extends ValidReferences - ? FindInvalidReferences - : FindInvalidReferences - : Invalid - : Invalid - : Invalid; - -export type ValidateRelationshipReferences< - FK extends { references: readonly string[] }, - ValidReferences extends string, +export type ValidateColumnsMatch< + ReferenceColumn extends AnyColumnSchemaComponent, + Column extends AnyColumnSchemaComponent, + references extends SchemaColumnName = SchemaColumnName, > = - AllInTuple extends true - ? ValidationResult - : ValidationResult< - false, - `Invalid foreign key references: ${FindInvalidReferences extends infer Invalid ? (Invalid extends string[] ? Invalid[number] : never) : never}. Available references: ${ValidReferences}` - >; + Column extends ColumnSchemaComponent + ? ReferenceColumn extends ColumnSchemaComponent + ? ValidateColumnTypeMatch + : never + : never; -export type ValidateSingleRelationship< - FK extends { columns: readonly string[]; references: readonly string[] }, - TableColumns extends string, - ValidReferences extends string, +export type ValidateReference< + RefPath extends SchemaColumnName = SchemaColumnName, + ColPath extends SchemaColumnName = SchemaColumnName, + Schemas extends DatabaseSchemas = DatabaseSchemas, > = - ValidateRelationshipLength extends { valid: false; error: infer E } - ? ValidationResult - : ValidateRelationshipColumns extends { - valid: false; - error: infer E; - } - ? ValidationResult - : ValidateRelationshipReferences extends { - valid: false; - error: infer E; - } - ? ValidationResult - : ValidationResult; + ColPath extends SchemaColumnName< + infer SchemaName, + infer TableName, + infer Column + > + ? ValidateColumnReference extends infer RefColumn + ? RefColumn extends AnyColumnSchemaComponent + ? ValidateColumnsMatch< + RefColumn, + Schemas[SchemaName]['tables'][TableName]['columns'][Column], + RefPath + > + : RefColumn extends { + valid: false; + error: infer E; + } + ? TypeValidationError + : never + : never + : never; -export type ValidateRelationship< - FKs extends Record, - TableColumns extends string, - ValidReferences extends string, -> = keyof FKs extends never - ? ValidationResult - : ValidateSingleRelationship< - FKs[keyof FKs], - TableColumns, - ValidReferences - > extends { - valid: false; - error: infer E; +export type ValidateReferences< + RefPath extends SchemaColumnName = SchemaColumnName, + ColPath extends SchemaColumnName = SchemaColumnName, + Schemas extends DatabaseSchemas = DatabaseSchemas, +> = + ColPath extends SchemaColumnName< + infer SchemaName, + infer TableName, + infer Column + > + ? ValidateColumnReference extends infer RefColumn + ? RefColumn extends AnyColumnSchemaComponent + ? ValidateColumnsMatch< + RefColumn, + Schemas[SchemaName]['tables'][TableName]['columns'][Column], + RefPath + > + : RefColumn extends { + valid: false; + error: infer E; + } + ? TypeValidationError + : never + : never + : never; + +export type CollectReferencesErrors< + Columns extends readonly SchemaColumnName[], + References extends readonly SchemaColumnName[], + _CurrentSchema extends string, + _CurrentTable extends string, + Schemas extends DatabaseSchemas = DatabaseSchemas, + Errors extends AnyTypeValidationError[] = [], +> = ZipTuplesCollectErrors< + References, + Columns, + { + [R in References[number]]: { + [C in Columns[number]]: ValidateReference; + }; + }, + Errors +>; + +export type SchemaTablesWithSingle
= + Table extends TableSchemaComponent< + infer _Columns, + infer TableName, + infer _FKs + > + ? DatabaseSchemaSchemaComponent<{ + [K in TableName]: Table; + }> + : never; + +export type DatabaseSchemasWithSingle< + Schema extends AnyDatabaseSchemaSchemaComponent, +> = + Schema extends DatabaseSchemaSchemaComponent + ? { + [K in _SchemaName]: Schema; } - ? ValidationResult - : ValidationResult; + : never; + +export type ValidateRelationship< + Columns extends TableColumns, + Relationship extends AnyTableRelationshipDefinitionWithColumns< + Extract + >, + RelationshipName extends string, + CurrentTableName extends string, + Table extends AnyTableSchemaComponent = AnyTableSchemaComponent, + Schema extends + AnyDatabaseSchemaSchemaComponent = SchemaTablesWithSingle
, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, +> = + FailOnFirstTypeValidationError< + [ + ValidateRelationshipLength, + ValidateRelationshipColumns, + CollectReferencesErrors< + NormalizeColumnPath< + Relationship['columns'], + Schema['schemaName'], + CurrentTableName + >, + NormalizeColumnPath< + Relationship['references'], + Schema['schemaName'], + CurrentTableName + >, + Schema['schemaName'], + CurrentTableName, + Schemas + > extends infer Results extends readonly AnyTypeValidationError[] + ? IF< + AnyTypeValidationFailed, + TypeValidationError>, + TypeValidationSuccess + > + : TypeValidationSuccess, + ] + > extends infer Error extends AnyTypeValidationError + ? TypeValidationError<{ + relationship: RelationshipName; + errors: Error extends TypeValidationError + ? E extends readonly unknown[] + ? E + : [E] + : never; + }> + : TypeValidationSuccess; + +export type CollectRelationshipErrors< + Columns extends TableColumns = TableColumns, + Relationships extends TableRelationships< + keyof Columns & string + > = {} & TableRelationships, + Table extends AnyTableSchemaComponent = AnyTableSchemaComponent, + Schema extends + AnyDatabaseSchemaSchemaComponent = SchemaTablesWithSingle
, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, + Errors extends AnyTypeValidationError[] = [], +> = MapRecordCollectErrors< + Relationships, + { + [R in keyof Relationships]: ValidateRelationship< + Columns, + Relationships[R] extends AnyTableRelationshipDefinitionWithColumns< + Extract + > + ? Relationships[R] + : never, + Extract, + Table['tableName'], + Table, + Schema, + Schemas + >; + }, + Errors +>; export type ValidateTableRelationships< Table extends AnyTableSchemaComponent, - ValidReferences extends string, + Schema extends + AnyDatabaseSchemaSchemaComponent = SchemaTablesWithSingle
, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, > = - Table extends TableSchemaComponent - ? ValidateRelationship< - FKs, - TableColumnNames
& string, - ValidReferences - > - : ValidationResult; + Table extends TableSchemaComponent< + infer Columns, + infer TableName, + infer Relationships + > + ? keyof Relationships extends Extract + ? CollectRelationshipErrors< + Columns, + Relationships, + Table, + Schema, + Schemas + > extends infer Results + ? AnyTypeValidationFailed extends true + ? TypeValidationError<{ + table: TableName; + errors: UnwrapTypeValidationErrors< + Results extends readonly AnyTypeValidationError[] + ? Results + : never + >; + }> + : Results + : TypeValidationSuccess + : TypeValidationSuccess + : TypeValidationSuccess; + +export type ValidateTable< + Table extends AnyTableSchemaComponent, + Schema extends + AnyDatabaseSchemaSchemaComponent = SchemaTablesWithSingle
, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, +> = ValidateTableRelationships; -export type ValidateTablesInSchema< +export type ValidateSchemaTables< Tables extends Record, - ValidReferences extends string, -> = { - [TableName in keyof Tables]: ValidateTableRelationships< - Tables[TableName], - ValidReferences - >; -}[keyof Tables] extends infer Results - ? Results extends { valid: true } - ? ValidationResult - : Results extends { valid: false; error: infer E } - ? ValidationResult - : ValidationResult - : ValidationResult; - -export type ValidateSchemaRelationships< + SchemaName extends string, Schema extends AnyDatabaseSchemaSchemaComponent, - ValidReferences extends string, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, > = - Schema extends DatabaseSchemaSchemaComponent - ? ValidateTablesInSchema - : ValidationResult; + MapRecordCollectErrors< + Tables, + { + [TableName in keyof Tables]: ValidateTable< + Tables[TableName], + Schema, + Schemas + >; + } + > extends infer Results + ? AnyTypeValidationFailed extends true + ? TypeValidationError<{ + schema: SchemaName; + errors: UnwrapTypeValidationErrors< + Results extends readonly AnyTypeValidationError[] ? Results : never + >; + }> + : TypeValidationSuccess + : TypeValidationSuccess; -export type ValidateSchemasInDatabase< - Schemas extends Record, - ValidReferences extends string, -> = { - [SchemaName in keyof Schemas]: ValidateSchemaRelationships< - Schemas[SchemaName], - ValidReferences - >; -}[keyof Schemas] extends infer Results - ? Results extends { valid: true } - ? ValidationResult - : Results extends { valid: false; error: infer E } - ? ValidationResult - : ValidationResult - : ValidationResult; - -export type ValidateDatabaseRelationships< - DB extends AnyDatabaseSchemaComponent, +export type ValidateDatabaseSchema< + Schema extends AnyDatabaseSchemaSchemaComponent, + Schemas extends DatabaseSchemas = DatabaseSchemasWithSingle, > = - DB extends DatabaseSchemaComponent - ? ValidateSchemasInDatabase> - : ValidationResult; + Schema extends DatabaseSchemaSchemaComponent + ? ValidateSchemaTables + : TypeValidationSuccess; + +export type ValidateDatabaseSchemas = + MapRecordCollectErrors< + Schemas, + { + [SchemaName in keyof Schemas]: ValidateDatabaseSchema< + Schemas[SchemaName], + Schemas + >; + } + > extends infer Results + ? AnyTypeValidationFailed extends true + ? TypeValidationError< + UnwrapTypeValidationErrors< + Results extends readonly AnyTypeValidationError[] ? Results : never + > + > + : TypeValidationSuccess + : TypeValidationSuccess; + +export type ValidateDatabaseSchemasWithMessages< + Schemas extends DatabaseSchemas, +> = FormatValidationErrors>; + +// TODO: Use in DatabaseSchema schema component validation +// export type ValidatedSchemaComponent< +// Tables extends DatabaseSchemaTables, +// SchemaName extends string, +// > = +// ValidateDatabaseSchema< +// DatabaseSchemaSchemaComponent, +// { schemaName: DatabaseSchemaSchemaComponent } +// > extends { +// valid: true; +// } +// ? DatabaseSchemaSchemaComponent +// : ValidateDatabaseSchema< +// DatabaseSchemaSchemaComponent, +// { schemaName: DatabaseSchemaSchemaComponent } +// > extends { +// valid: false; +// error: infer E; +// } +// ? { valid: false; error: FormatError } +// : DatabaseSchemaSchemaComponent; diff --git a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts deleted file mode 100644 index baa6fad8..00000000 --- a/src/packages/dumbo/src/core/schema/components/relationships/relationshipValidation.type.spec.ts +++ /dev/null @@ -1,650 +0,0 @@ -import type { - AnyColumnSchemaComponent, - AnyDatabaseSchemaComponent, - AnyDatabaseSchemaSchemaComponent, - AnyTableSchemaComponent, - DatabaseSchemaComponent, - DatabaseSchemaSchemaComponent, - TableSchemaComponent, -} from '..'; -import { SQL } from '../../../sql'; -import type { Equal, Expect } from '../../../testing'; -import { dumboSchema } from '../../dumboSchema'; -import type { - AllColumnReferences, - ExtractColumnNames, - ExtractSchemaNames, - ExtractTableNames, - RelationshipDefinition, -} from './relationshipTypes'; - -const { database, schema, table, column } = dumboSchema; -const { Varchar } = SQL.column.type; - -type _DB1 = DatabaseSchemaComponent<{ - public: AnyDatabaseSchemaSchemaComponent; -}>; -type _Test1 = Expect, 'public'>>; - -type _DB2 = DatabaseSchemaComponent<{ - public: AnyDatabaseSchemaSchemaComponent; - analytics: AnyDatabaseSchemaSchemaComponent; -}>; -type _Test2 = Expect, 'public' | 'analytics'>>; - -type _Schema1 = DatabaseSchemaSchemaComponent<{ - users: AnyTableSchemaComponent; -}>; -type _Test3 = Expect, 'users'>>; - -type _Schema2 = DatabaseSchemaSchemaComponent<{ - users: AnyTableSchemaComponent; - posts: AnyTableSchemaComponent; - comments: AnyTableSchemaComponent; -}>; -type _Test4 = Expect< - Equal, 'users' | 'posts' | 'comments'> ->; - -type _Table1 = TableSchemaComponent<{ - id: AnyColumnSchemaComponent; -}>; -type _Test5 = Expect, 'id'>>; - -type _Table2 = TableSchemaComponent<{ - id: AnyColumnSchemaComponent; - email: AnyColumnSchemaComponent; - name: AnyColumnSchemaComponent; - created_at: AnyColumnSchemaComponent; -}>; -type _Test6 = Expect< - Equal, 'id' | 'email' | 'name' | 'created_at'> ->; - -const _db1 = database('test', { - public: schema('public', { - users: table('users', { - columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), - }, - }), - }), -}); - -type _Result1 = AllColumnReferences; -type _Test7 = Expect>; - -const _db2 = database('test', { - public: schema('public', { - users: table('users', { - columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), - }, - }), - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - title: column('title', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - }), - }), -}); - -type _Result2 = AllColumnReferences; -type _Test8 = Expect< - Equal< - _Result2, - | 'public.users.id' - | 'public.users.email' - | 'public.posts.id' - | 'public.posts.title' - | 'public.posts.user_id' - > ->; - -const _db3 = database('test', { - public: schema('public', { - users: table('users', { - columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), - }, - }), - }), - analytics: schema('analytics', { - events: table('events', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - event_type: column('event_type', Varchar('max')), - }, - }), - }), -}); - -type _Result3 = AllColumnReferences; -type _Test9 = Expect< - Equal< - _Result3, - | 'public.users.id' - | 'public.users.email' - | 'analytics.events.id' - | 'analytics.events.user_id' - | 'analytics.events.event_type' - > ->; - -const _validFK: RelationshipDefinition = { - columns: ['user_id'], - references: ['public.users.id'], - type: 'one-to-one', -}; - -type _ColumnsType = typeof _validFK.columns; -type _Test10 = Expect>; - -const _compositeFK: RelationshipDefinition = { - columns: ['user_id', 'tenant_id'], - references: ['public.users.id', 'public.users.tenant_id'], - type: 'one-to-one', -}; - -type _CompositeColumnsType = typeof _compositeFK.columns; -type _CompositeReferencesType = typeof _compositeFK.references; -type _Test11 = Expect>; -type _Test12 = Expect>; - -import type { IsError } from '../../../testing/typesTesting'; -import type { - ValidateDatabaseRelationships, - ValidateRelationshipLength, -} from './relationshipValidation'; - -type _FK_LengthMismatch = { - columns: ['user_id', 'tenant_id']; - references: ['public.users.id']; -}; - -type _Result_LengthMismatch = ValidateRelationshipLength<_FK_LengthMismatch>; -type _Test13 = Expect>; - -type _FK_SingleMatch = { - columns: ['user_id']; - references: ['public.users.id']; -}; - -type _FK_CompositeMatch = { - columns: ['user_id', 'tenant_id']; - references: ['public.users.id', 'public.users.tenant_id']; -}; - -type _Result_SingleMatch = ValidateRelationshipLength<_FK_SingleMatch>; -type _Result_CompositeMatch = ValidateRelationshipLength<_FK_CompositeMatch>; -type _Test14 = Expect>; -type _Test15 = Expect>; - -import type { ValidateRelationshipColumns } from './relationshipValidation'; - -type _FK_InvalidColumn = { - columns: ['user_id', 'invalid_col']; - references: ['public.users.id', 'public.users.tenant_id']; -}; - -type _Result_InvalidColumn = ValidateRelationshipColumns< - _FK_InvalidColumn, - 'id' | 'email' | 'user_id' ->; -type _Test16 = Expect>; - -type _FK_ValidColumns = { - columns: ['user_id']; - references: ['public.users.id']; -}; - -type _FK_ValidCompositeColumns = { - columns: ['user_id', 'email']; - references: ['public.users.id', 'public.users.email']; -}; - -type _Result_ValidColumns = ValidateRelationshipColumns< - _FK_ValidColumns, - 'id' | 'email' | 'user_id' ->; -type _Result_ValidCompositeColumns = ValidateRelationshipColumns< - _FK_ValidCompositeColumns, - 'id' | 'email' | 'user_id' ->; -type _Test17 = Expect>; -type _Test18 = Expect>; - -import type { ValidateRelationshipReferences } from './relationshipValidation'; - -type _FK_InvalidReference = { - columns: ['user_id']; - references: ['public.nonexistent.id']; -}; - -type _Result_InvalidReference = ValidateRelationshipReferences< - _FK_InvalidReference, - 'public.users.id' | 'public.users.email' | 'public.posts.id' ->; -type _Test19 = Expect>; - -type _FK_ValidReference = { - columns: ['user_id']; - references: ['public.users.id']; -}; - -type _FK_ValidCompositeReference = { - columns: ['user_id', 'post_id']; - references: ['public.users.id', 'public.posts.id']; -}; - -type _Result_ValidReference = ValidateRelationshipReferences< - _FK_ValidReference, - 'public.users.id' | 'public.users.email' | 'public.posts.id' ->; -type _Result_ValidCompositeReference = ValidateRelationshipReferences< - _FK_ValidCompositeReference, - 'public.users.id' | 'public.users.email' | 'public.posts.id' ->; -type _Test20 = Expect>; -type _Test21 = Expect>; - -import type { ValidateSingleRelationship } from './relationshipValidation'; - -type _FK_Complete_Valid = { - columns: ['user_id']; - references: ['public.users.id']; -}; - -type _Result_Complete_Valid = ValidateSingleRelationship< - _FK_Complete_Valid, - 'id' | 'user_id', - 'public.users.id' | 'public.users.email' ->; -type _Test22 = Expect>; - -type _FK_Complete_LengthError = { - columns: ['user_id', 'tenant_id']; - references: ['public.users.id']; -}; - -type _Result_Complete_LengthError = ValidateSingleRelationship< - _FK_Complete_LengthError, - 'id' | 'user_id' | 'tenant_id', - 'public.users.id' | 'public.users.email' ->; -type _Test23 = Expect>; - -type _FK_Complete_ColumnError = { - columns: ['invalid_col']; - references: ['public.users.id']; -}; - -type _Result_Complete_ColumnError = ValidateSingleRelationship< - _FK_Complete_ColumnError, - 'id' | 'user_id', - 'public.users.id' | 'public.users.email' ->; -type _Test24 = Expect>; - -type _FK_Complete_ReferenceError = { - columns: ['user_id']; - references: ['public.invalid.id']; -}; - -type _Result_Complete_ReferenceError = ValidateSingleRelationship< - _FK_Complete_ReferenceError, - 'id' | 'user_id', - 'public.users.id' | 'public.users.email' ->; -type _Test24A = Expect< - Equal< - _Result_Complete_ReferenceError, - { - valid: false; - error: - | 'Invalid foreign key references: public.invalid.id. Available references: public.users.id' - | 'Invalid foreign key references: public.invalid.id. Available references: public.users.email'; - } - > ->; - -import type { ValidateRelationship } from './relationshipValidation'; - -type _FKRecord_Mixed = { - user_fk: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - invalid_fk: { - columns: ['invalid_col']; - references: ['public.users.email']; - type: 'one-to-many'; - }; -}; - -type _Result_FKRecord_Mixed = ValidateRelationship< - _FKRecord_Mixed, - 'id' | 'user_id', - 'public.users.id' | 'public.users.email' ->; - -type _Test25A = Expect< - Equal< - _Result_FKRecord_Mixed, - { - valid: false; - error: - | 'Invalid foreign key columns: invalid_col. Available columns: user_id' - | 'Invalid foreign key columns: invalid_col. Available columns: id'; - } - > ->; -type _Test35 = Expect>; - -type _FKRecord_AllValid = { - user_fk: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - email_fk: { - columns: ['email']; - references: ['public.users.email']; - type: 'one-to-many'; - }; -}; - -type _Result_FKRecord_AllValid = ValidateRelationship< - _FKRecord_AllValid, - 'id' | 'user_id' | 'email', - 'public.users.id' | 'public.users.email' ->; -type _ValidateRelationshipRecordResult_InvalidFK = ValidateRelationship< - { - invalid_fk: { - columns: ['invalid']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - }, - 'id' | 'user_id', - 'public.users.id' | 'public.users.email' ->; -type _TestValidateRelationshipRecordResult_InvalidFK = Expect< - IsError<_ValidateRelationshipRecordResult_InvalidFK> ->; - -import type { ValidateTableRelationships } from './relationshipValidation'; - -type _Table_NoFKs = TableSchemaComponent<{ - id: AnyColumnSchemaComponent; - email: AnyColumnSchemaComponent; -}>; - -type _Result_NoFKs = ValidateTableRelationships< - _Table_NoFKs, - 'public.users.id' | 'public.users.email' ->; -type _Test26 = Expect>; - -type _Table_SingleFK = TableSchemaComponent< - { - id: AnyColumnSchemaComponent; - user_id: AnyColumnSchemaComponent; - }, - { - author: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - } ->; - -type _Result_SingleFK = ValidateTableRelationships< - _Table_SingleFK, - 'public.users.id' | 'public.users.email' ->; -type _Test27 = Expect>; - -type _Table_MultipleFK = TableSchemaComponent< - { - id: AnyColumnSchemaComponent; - user_id: AnyColumnSchemaComponent; - author_id: AnyColumnSchemaComponent; - }, - { - user: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - author: { - columns: ['author_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - } ->; - -type _Result_MultipleFK = ValidateTableRelationships< - _Table_MultipleFK, - 'public.users.id' | 'public.users.email' ->; -type _Test28 = Expect>; - -type _Table_InvalidFK = TableSchemaComponent< - { - id: AnyColumnSchemaComponent; - user_id: AnyColumnSchemaComponent; - }, - { - user: { - readonly columns: ['id']; - references: readonly ['public.users.id']; - type: 'one-to-many'; - }; - } ->; - -type _Result_InvalidFK = ValidateTableRelationships< - _Table_InvalidFK, - 'public.posts.id' | 'public.users.email' ->; -type _Test29 = Expect>; - -import type { ValidateSchemaRelationships } from './relationshipValidation'; - -type _Schema_MultiTable = DatabaseSchemaSchemaComponent<{ - users: TableSchemaComponent<{ - id: AnyColumnSchemaComponent; - email: AnyColumnSchemaComponent; - }>; - posts: TableSchemaComponent< - { - id: AnyColumnSchemaComponent; - user_id: AnyColumnSchemaComponent; - }, - { - user: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - } - >; -}>; - -type _Result_Schema_Valid = ValidateSchemaRelationships< - _Schema_MultiTable, - 'public.users.id' | 'public.users.email' | 'public.posts.id' ->; -type _Test30 = Expect>; - -type _Schema_WithError = DatabaseSchemaSchemaComponent<{ - posts: TableSchemaComponent< - { - id: AnyColumnSchemaComponent; - user_id: AnyColumnSchemaComponent; - }, - { - user: { - columns: ['user_id']; - references: ['public.users.id']; - type: 'one-to-many'; - }; - } - >; -}>; - -type _Result_Schema_Error = ValidateSchemaRelationships< - _Schema_WithError, - 'public.posts.id' | 'public.users.email' ->; -type _Test31 = Expect>; - -const _dbWithErrorVSInDB = database('test', { - public: schema('public', { - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - relationships: { - invalid: { - columns: ['id'], - references: ['public.users.id'], - type: 'one-to-many', - }, - }, - }), - }), -}); - -type _Test_ValidateSchemasInDatabaseResult_DbError = Expect< - IsError ->; - -const _fullDb = database('test', { - public: schema('public', { - users: table('users', { - columns: { - id: column('id', Varchar('max')), - email: column('email', Varchar('max')), - }, - }), - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - relationships: { - user: { - type: 'many-to-one', - columns: ['user_id'], - references: ['public.users.id'], - }, - }, - }), - }), -}); - -type _Result_FullDb = ValidateDatabaseRelationships; -type _Test32 = Expect>; - -const _dbWithSelfRef = database('test', { - public: schema('public', { - users: table('users', { - columns: { - id: column('id', Varchar('max')), - manager_id: column('manager_id', Varchar('max')), - }, - relationships: { - manager: { - columns: ['manager_id'], - references: ['public.users.id'], - type: 'many-to-one', - }, - }, - }), - }), -}); - -type _Result_SelfRef = ValidateDatabaseRelationships; -type _Test33 = Expect>; - -const _dbWithError = database('test', { - public: schema('public', { - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - relationships: { - invalid: { - columns: ['id'], - references: ['public.users.id'], - type: 'many-to-one', - }, - }, - }), - }), -}); - -type _Test34 = Expect>; - -// TEST: Invalid column should cause type error at database() call -const _dbInvalidColumn = database('test', { - public: schema('public', { - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('id', Varchar('max')), - }, - relationships: { - invalid: { - columns: ['id'], - references: ['public.users.id'], - type: 'many-to-one', - }, - }, - }), - }), -}); - -type _InvalidColResult = typeof _dbInvalidColumn; -type _Test_InvalidColumn = Expect>; - -// TEST: Valid FK should work -const _dbValid = database('test', { - public: schema('public', { - users: table('users', { - columns: { id: column('id', Varchar('max')) }, - }), - posts: table('posts', { - columns: { - id: column('id', Varchar('max')), - user_id: column('user_id', Varchar('max')), - }, - relationships: { - user: { - columns: ['user_id'], - references: ['public.users.id'], - type: 'many-to-one', - }, - }, - }), - }), -}); - -type _ValidResult = typeof _dbValid; -type _Test_Valid = Expect< - Equal<_ValidResult extends AnyDatabaseSchemaComponent ? true : false, true> ->; // This should PASS diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchema.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchema.type.spec.ts new file mode 100644 index 00000000..7ac8f440 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchema.type.spec.ts @@ -0,0 +1,187 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect, IsError } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import { relationship } from './relationshipTypes'; +import type { ValidateDatabaseSchema } from './relationshipValidation'; + +const { schema, table, column } = dumboSchema; +const { Integer, BigInteger } = SQL.column.type; + +void describe('ValidateDatabaseSchema', () => { + const usersTable = table('users', { + columns: { + id: column('id', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + post_id: column('post_id', Integer), + user_id: column('user_id', Integer), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'one-to-one'), + }, + }); + + const _validSchema = schema('public', { + users: usersTable, + posts: postsTable, + }); + + type ValidSchemas = { + public: typeof _validSchema; + }; + + void it('returns success when all tables are valid', () => { + type Result = ValidateDatabaseSchema; + + type _Then = Expect>>; + }); + + void it('collects errors from a single invalid table', () => { + const invalidTable = table('invalid', { + columns: { + col1: column('col1', Integer), + col2: column('col2', Integer), + }, + relationships: { + bad_rel: relationship( + ['col1', 'col2'], + ['public.users.id'], + 'one-to-one', + ), + }, + }); + + const _schemaWithInvalidTable = schema('public', { + users: usersTable, + invalid: invalidTable, + }); + + type TestSchemas = { + public: typeof _schemaWithInvalidTable; + }; + + type Result = ValidateDatabaseSchema< + typeof _schemaWithInvalidTable, + TestSchemas + >; + + type Expected = TypeValidationResult< + false, + { + schema: 'public'; + errors: [ + { + table: 'invalid'; + errors: [ + { + relationship: 'bad_rel'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: readonly ['col1', 'col2']; + references: readonly ['public.users.id']; + }, + ]; + }, + ]; + }, + ]; + } + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('collects errors from multiple invalid tables', () => { + const invalidTable1 = table('invalid1', { + columns: { + col1: column('col1', Integer), + }, + relationships: { + bad_rel: relationship(['col1'], ['nonexistent.table.id'], 'one-to-one'), + }, + }); + + const invalidTable2 = table('invalid2', { + columns: { + col2: column('col2', BigInteger), + }, + relationships: { + user: relationship(['col2'], ['public.users.id'], 'one-to-one'), + }, + }); + + const _schemaWithMultipleInvalid = schema('public', { + users: usersTable, + invalid1: invalidTable1, + invalid2: invalidTable2, + }); + + type TestSchemas = { + public: typeof _schemaWithMultipleInvalid; + }; + + type Result = ValidateDatabaseSchema< + typeof _schemaWithMultipleInvalid, + TestSchemas + >; + + type Expected = TypeValidationResult< + false, + { + schema: 'public'; + errors: [ + { + table: 'invalid1'; + errors: [ + { + relationship: 'bad_rel'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.table.id'; + }, + ]; + }, + ]; + }, + { + table: 'invalid2'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + ]; + }, + ]; + } + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('returns success when schema has no tables', () => { + const _emptySchema = schema('empty', {}); + + type EmptySchemas = { + empty: typeof _emptySchema; + }; + + type Result = ValidateDatabaseSchema; + + type _Then = Expect>>; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchemas.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchemas.type.spec.ts new file mode 100644 index 00000000..05d9efcd --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateDatabaseSchemas.type.spec.ts @@ -0,0 +1,190 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect, IsError } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import { relationship } from './relationshipTypes'; +import type { ValidateDatabaseSchemas } from './relationshipValidation'; + +const { schema, table, column } = dumboSchema; +const { Integer, BigInteger } = SQL.column.type; + +void describe('ValidateDatabaseSchemas', () => { + const usersTable = table('users', { + columns: { + id: column('id', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + post_id: column('post_id', Integer), + user_id: column('user_id', Integer), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'one-to-one'), + }, + }); + + void it('returns success when all schemas are valid', () => { + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + }); + + type ValidSchemas = { + public: typeof _publicSchema; + }; + + type Result = ValidateDatabaseSchemas; + + type _Then = Expect>>; + }); + + void it('collects errors from a single invalid schema', () => { + const invalidTable = table('invalid', { + columns: { + col1: column('col1', Integer), + col2: column('col2', Integer), + }, + relationships: { + bad_rel: relationship( + ['col1', 'col2'], + ['public.users.id'], + 'one-to-one', + ), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + invalid: invalidTable, + }); + + type TestSchemas = { + public: typeof _publicSchema; + }; + + type Result = ValidateDatabaseSchemas; + + type Expected = TypeValidationResult< + false, + [ + { + schema: 'public'; + errors: [ + { + table: 'invalid'; + errors: [ + { + relationship: 'bad_rel'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: readonly ['col1', 'col2']; + references: readonly ['public.users.id']; + }, + ]; + }, + ]; + }, + ]; + }, + ] + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('collects errors from multiple invalid schemas', () => { + const invalidTable1 = table('invalid1', { + columns: { + col1: column('col1', Integer), + }, + relationships: { + bad_rel: relationship(['col1'], ['nonexistent.table.id'], 'one-to-one'), + }, + }); + + const _schema1 = schema('schema1', { + invalid1: invalidTable1, + }); + + const invalidTable2 = table('invalid2', { + columns: { + col2: column('col2', BigInteger), + }, + relationships: { + user: relationship(['col2'], ['schema1.invalid1.col1'], 'one-to-one'), + }, + }); + + const _schema2 = schema('schema2', { + invalid2: invalidTable2, + }); + + type TestSchemas = { + schema1: typeof _schema1; + schema2: typeof _schema2; + }; + + type Result = ValidateDatabaseSchemas; + + type Expected = TypeValidationResult< + false, + [ + { + schema: 'schema1'; + errors: [ + { + table: 'invalid1'; + errors: [ + { + relationship: 'bad_rel'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.table.id'; + }, + ]; + }, + ]; + }, + ]; + }, + { + schema: 'schema2'; + errors: [ + { + table: 'invalid2'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'schema1.invalid1.col1'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + ]; + }, + ]; + }, + ] + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('returns success when database has no schemas', () => { + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + type EmptySchemas = {}; + + type Result = ValidateDatabaseSchemas; + + type _Then = Expect>>; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateReference.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateReference.type.spec.ts new file mode 100644 index 00000000..f42b1dff --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateReference.type.spec.ts @@ -0,0 +1,365 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect, IsError } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import type { SchemaColumnName } from './relationshipTypes'; +import type { ValidateReference } from './relationshipValidation'; + +const { column, table, schema } = dumboSchema; +const { BigInteger, Varchar, Integer } = SQL.column.type; + +void describe('ValidateReference', () => { + const usersTable = table('users', { + columns: { + id: column('id', BigInteger), + name: column('name', Varchar('max')), + age: column('age', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + post_id: column('post_id', BigInteger), + user_id: column('user_id', BigInteger), + title: column('title', Varchar('max')), + view_count: column('view_count', Integer), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + }); + + type TestSchemas = { + public: typeof _publicSchema; + }; + + void describe('reference existence validation', () => { + void it('fails when referenced schema does not exist', () => { + type RefPath = SchemaColumnName<'nonexistent', 'users', 'id'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + } + > + > + >, + ]; + }); + + void it('fails when referenced table does not exist', () => { + type RefPath = SchemaColumnName<'public', 'nonexistent', 'id'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'missing_table'; + reference: 'public.nonexistent.id'; + } + > + > + >, + ]; + }); + + void it('fails when referenced column does not exist', () => { + type RefPath = SchemaColumnName<'public', 'users', 'nonexistent'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'missing_column'; + reference: 'public.users.nonexistent'; + } + > + > + >, + ]; + }); + }); + + void describe('type matching: both ColumnTypeToken', () => { + void it('validates when types match', () => { + type RefPath = SchemaColumnName<'public', 'users', 'id'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + + void it('fails when types do not match', () => { + type RefPath = SchemaColumnName<'public', 'users', 'name'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'type_mismatch'; + reference: 'public.users.name'; + referenceType: 'VARCHAR'; + columnTypeName: 'BIGINT'; + } + > + > + >, + ]; + }); + + void it('fails when integer does not match bigint', () => { + type RefPath = SchemaColumnName<'public', 'users', 'age'>; + type ColPath = SchemaColumnName<'public', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'type_mismatch'; + reference: 'public.users.age'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + } + > + > + >, + ]; + }); + }); + + void describe('type matching: ColumnType is ColumnTypeToken, RefColumnType is string', () => { + const stringRefTable = table('string_ref', { + columns: { + id: column('id', 'BIGINT'), + label: column('label', 'VARCHAR'), + }, + }); + + const _mixedSchema1 = schema('mixed1', { + string_ref: stringRefTable, + posts: postsTable, + }); + + type MixedSchemas1 = { + mixed1: typeof _mixedSchema1; + }; + + void it('validates when string reference type matches ColumnTypeToken', () => { + type RefPath = SchemaColumnName<'mixed1', 'string_ref', 'id'>; + type ColPath = SchemaColumnName<'mixed1', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + + void it('fails when string reference type does not match ColumnTypeToken', () => { + type RefPath = SchemaColumnName<'mixed1', 'string_ref', 'label'>; + type ColPath = SchemaColumnName<'mixed1', 'posts', 'user_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'type_mismatch'; + reference: 'mixed1.string_ref.label'; + referenceType: 'VARCHAR'; + columnTypeName: 'BIGINT'; + } + > + > + >, + ]; + }); + }); + + void describe('type matching: RefColumnType is ColumnTypeToken, ColumnType is string', () => { + const stringColTable = table('string_col', { + columns: { + id: column('id', 'BIGINT'), + count: column('count', 'INTEGER'), + }, + }); + + const _mixedSchema2 = schema('mixed2', { + string_col: stringColTable, + users: usersTable, + }); + + type MixedSchemas2 = { + mixed2: typeof _mixedSchema2; + }; + + void it('validates when ColumnTypeToken reference matches string type', () => { + type RefPath = SchemaColumnName<'mixed2', 'users', 'id'>; + type ColPath = SchemaColumnName<'mixed2', 'string_col', 'id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + + void it('fails when ColumnTypeToken reference does not match string type', () => { + type RefPath = SchemaColumnName<'mixed2', 'users', 'age'>; + type ColPath = SchemaColumnName<'mixed2', 'string_col', 'id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'type_mismatch'; + reference: 'mixed2.users.age'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + } + > + > + >, + ]; + }); + }); + + void describe('type matching: both are strings', () => { + const stringOnlyTable1 = table('string_only1', { + columns: { + id: column('id', 'BIGINT'), + label: column('label', 'VARCHAR'), + }, + }); + + const stringOnlyTable2 = table('string_only2', { + columns: { + ref_id: column('ref_id', 'BIGINT'), + description: column('description', 'VARCHAR'), + }, + }); + + const _mixedSchema3 = schema('mixed3', { + string_only1: stringOnlyTable1, + string_only2: stringOnlyTable2, + }); + + type MixedSchemas3 = { + mixed3: typeof _mixedSchema3; + }; + + void it('validates when both string types match', () => { + type RefPath = SchemaColumnName<'mixed3', 'string_only1', 'id'>; + type ColPath = SchemaColumnName<'mixed3', 'string_only2', 'ref_id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + + void it('fails when both string types do not match', () => { + type RefPath = SchemaColumnName<'mixed3', 'string_only1', 'label'>; + type ColPath = SchemaColumnName<'mixed3', 'string_only2', 'ref_id'>; + + type Result = ValidateReference; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'type_mismatch'; + reference: 'mixed3.string_only1.label'; + referenceType: 'VARCHAR'; + columnTypeName: 'BIGINT'; + } + > + > + >, + ]; + }); + }); + + void describe('edge cases', () => { + void it('validates self-referencing column', () => { + type RefPath = SchemaColumnName<'public', 'users', 'id'>; + type ColPath = SchemaColumnName<'public', 'users', 'id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + + void it('validates reference across different tables with same type', () => { + type RefPath = SchemaColumnName<'public', 'posts', 'post_id'>; + type ColPath = SchemaColumnName<'public', 'users', 'id'>; + + type Result = ValidateReference; + + type _Then = Expect< + Equals> + >; + }); + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateRelationship.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationship.type.spec.ts new file mode 100644 index 00000000..d8d663b6 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationship.type.spec.ts @@ -0,0 +1,421 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect, IsError } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import type { InferTableSchemaComponentColumns } from '../tableSchemaComponent'; +import type { ValidateRelationship } from './relationshipValidation'; + +const { schema, table, column } = dumboSchema; +const { Integer, BigInteger } = SQL.column.type; + +void describe('ValidateRelationship', () => { + const usersTable = table('users', { + columns: { + id: column('id', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + post_id: column('post_id', Integer), + user_id: column('user_id', Integer), + tenant_id: column('tenant_id', Integer), + }, + }); + + const typeMismatchTable = table('type_mismatch', { + columns: { + user_id: column('user_id', BigInteger), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + type_mismatch: typeMismatchTable, + }); + + type _TestSchemas = { + public: typeof _publicSchema; + }; + + type PostsTable = typeof postsTable; + type ExistingColumns = InferTableSchemaComponentColumns; + + void it('fails when columns and references have different lengths', () => { + type MismatchedLengthRel = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + MismatchedLengthRel, + 'user_author', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'user_author'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; + }, + ]; + } + > + > + >, + ]; + }); + + void it('fails when columns and references are both empty', () => { + type EmptyRel = { + columns: []; + references: []; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + EmptyRel, + 'empty_rel', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'empty_rel'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: []; + references: []; + }, + ]; + } + > + > + >, + ]; + }); + + void it('fails when references are longer than columns', () => { + type ReferencesLongerRel = { + columns: ['user_id']; + references: ['public.users.id', 'public.users.tenant_id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + ReferencesLongerRel, + 'multi_ref', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'multi_ref'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: ['user_id']; + references: ['public.users.id', 'public.users.tenant_id']; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects missing schema errors', () => { + type MissingSchemaRel = { + columns: ['user_id']; + references: ['nonexistent.users.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + MissingSchemaRel, + 'bad_schema_ref', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'bad_schema_ref'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects missing table errors', () => { + type MissingTableRel = { + columns: ['user_id']; + references: ['public.nonexistent.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + MissingTableRel, + 'bad_table_ref', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'bad_table_ref'; + errors: [ + { + errorCode: 'missing_table'; + reference: 'public.nonexistent.id'; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects missing column errors', () => { + type MissingColumnRel = { + columns: ['user_id']; + references: ['public.users.nonexistent']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + MissingColumnRel, + 'bad_column_ref', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'bad_column_ref'; + errors: [ + { + errorCode: 'missing_column'; + reference: 'public.users.nonexistent'; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects multiple errors from different references', () => { + type MultipleErrorsRel = { + columns: ['user_id', 'tenant_id']; + references: ['nonexistent.users.id', 'public.missing_table.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + MultipleErrorsRel, + 'multi_error_rel', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'multi_error_rel'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + { + errorCode: 'missing_table'; + reference: 'public.missing_table.id'; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects all errors when all references are invalid', () => { + type AllInvalidRel = { + columns: ['user_id', 'tenant_id', 'post_id']; + references: [ + 'schema1.table1.col1', + 'schema2.table2.col2', + 'schema3.table3.col3', + ]; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + ExistingColumns, + AllInvalidRel, + 'all_invalid', + 'posts', + PostsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'all_invalid'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'schema1.table1.col1'; + }, + { + errorCode: 'missing_schema'; + reference: 'schema2.table2.col2'; + }, + { + errorCode: 'missing_schema'; + reference: 'schema3.table3.col3'; + }, + ]; + } + > + > + >, + ]; + }); + + void it('collects type mismatch errors', () => { + type TypeMismatchTable = typeof typeMismatchTable; + type TypeMismatchColumns = + InferTableSchemaComponentColumns; + + type TypeMismatchRel = { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationship< + TypeMismatchColumns, + TypeMismatchRel, + 'user', + 'type_mismatch', + TypeMismatchTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + } + > + > + >, + ]; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipColumns.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipColumns.type.spec.ts new file mode 100644 index 00000000..2bd54488 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipColumns.type.spec.ts @@ -0,0 +1,167 @@ +import { describe, it } from 'node:test'; +import type { Equals, Expect, IsError, IsOK } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import type { AnyColumnSchemaComponent } from '../columnSchemaComponent'; +import type { ValidateRelationshipColumns } from './relationshipValidation'; + +void describe('ValidateRelationshipColumns', () => { + type ExistingColumns = { + post_id: AnyColumnSchemaComponent; + user_id: AnyColumnSchemaComponent; + tenant_id: AnyColumnSchemaComponent; + }; + + void it('succeeds for single reference and column', () => { + type SingleColumnAndReferences = { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + SingleColumnAndReferences, + ExistingColumns + >; + + type _Then = Expect>; + }); + + void it('succeeds for multiple reference and column of the same length', () => { + type MultipleColumnsAndReferences = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id', 'public.users.tenant_id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + MultipleColumnsAndReferences, + ExistingColumns + >; + + type _Then = Expect>; + }); + + void it('fails when references and columns are empty', () => { + type EmptyRelationship = { + columns: []; + references: []; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + EmptyRelationship, + ExistingColumns + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_columns_mismatch'; + invalidColumns: []; + availableColumns: keyof ExistingColumns; + } + > + > + >, + ]; + }); + + void it('fails for single invalid columns', () => { + type SingleColumnAndReferences = { + columns: ['invalid']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + SingleColumnAndReferences, + ExistingColumns + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_columns_mismatch'; + invalidColumns: ['invalid']; + availableColumns: keyof ExistingColumns; + } + > + > + >, + ]; + }); + + void it('fails for multiple invalid columns', () => { + type MultipleColumnsAndReferences = { + columns: ['invalid', 'not_exist']; + references: ['public.users.id', 'public.users.tenant_id']; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + MultipleColumnsAndReferences, + ExistingColumns + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_columns_mismatch'; + invalidColumns: ['invalid', 'not_exist']; + availableColumns: keyof ExistingColumns; + } + > + > + >, + ]; + }); + + void it('fails for multiple invalid columns with a valid one', () => { + type MultipleColumnsAndReferences = { + columns: ['invalid', 'not_exist', 'user_id']; + references: [ + 'public.users.id', + 'public.users.tenant_id', + 'public.users.id', + ]; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipColumns< + MultipleColumnsAndReferences, + ExistingColumns + >; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_columns_mismatch'; + invalidColumns: ['invalid', 'not_exist']; + availableColumns: keyof ExistingColumns; + } + > + > + >, + ]; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipLength.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipLength.type.spec.ts new file mode 100644 index 00000000..eaad98f7 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateRelationshipLength.type.spec.ts @@ -0,0 +1,124 @@ +import { describe, it } from 'node:test'; +import type { Equals, Expect, IsError, IsOK } from '../../../testing'; +import type { + TypeValidationResult, + TypeValidationSuccess, +} from '../../../typing'; +import type { ValidateRelationshipLength } from './relationshipValidation'; + +void describe('ValidateRelationshipLength', () => { + void it('succeeds for single reference and column', () => { + type SingleColumnAndReferences = { + columns: ['user_id']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type _Result_LengthMismatch = + ValidateRelationshipLength; + + type _Then = [ + Expect>, + Expect>, + ]; + }); + + void it('succeeds for multiple reference and column of the same length', () => { + type MultipleColumnsAndReferences = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id', 'public.users.tenant_id']; + type: 'one-to-one'; + }; + + type _Result_LengthMismatch = + ValidateRelationshipLength; + + type _Assert = [ + Expect>, + Expect>, + ]; + }); + + void it('fails when references and columns are empty', () => { + type EmptyRelationship = { + columns: []; + references: []; + type: 'one-to-one'; + }; + + type Result = ValidateRelationshipLength; + + type _Assert = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_length_mismatch'; + columns: []; + references: []; + } + > + > + >, + ]; + }); + + void it('fails when columns are longer than references', () => { + type RelWithColumnsLongerThanReferences = { + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; + type: 'one-to-one'; + }; + + type Result = + ValidateRelationshipLength; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_length_mismatch'; + columns: ['user_id', 'tenant_id']; + references: ['public.users.id']; + } + > + > + >, + ]; + }); + + void it('fails when references are longer than columns', () => { + type RelWithReferencesLongerThanColumns = { + columns: ['user_id']; + references: ['public.users.id', 'public.users.tenant_id']; + type: 'one-to-one'; + }; + + type Result = + ValidateRelationshipLength; + + type _Then = [ + Expect>, + Expect< + Equals< + Result, + TypeValidationResult< + false, + { + errorCode: 'reference_length_mismatch'; + columns: ['user_id']; + references: ['public.users.id', 'public.users.tenant_id']; + } + > + > + >, + ]; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/relationships/validateTableRelationships.type.spec.ts b/src/packages/dumbo/src/core/schema/components/relationships/validateTableRelationships.type.spec.ts new file mode 100644 index 00000000..3e2047a3 --- /dev/null +++ b/src/packages/dumbo/src/core/schema/components/relationships/validateTableRelationships.type.spec.ts @@ -0,0 +1,236 @@ +import { describe, it } from 'node:test'; +import { SQL } from '../../../sql'; +import type { Equals, Expect, IsError } from '../../../testing'; +import type { TypeValidationResult } from '../../../typing'; +import { dumboSchema } from '../../dumboSchema'; +import { relationship } from './relationshipTypes'; +import type { ValidateTableRelationships } from './relationshipValidation'; + +const { schema, table, column } = dumboSchema; +const { Integer, BigInteger } = SQL.column.type; + +void describe('ValidateTableRelationships', () => { + const usersTable = table('users', { + columns: { + id: column('id', Integer), + }, + }); + + const postsTable = table('posts', { + columns: { + post_id: column('post_id', Integer), + user_id: column('user_id', Integer), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'one-to-one'), + }, + }); + + const typeMismatchTable = table('type_mismatch', { + columns: { + user_id: column('user_id', BigInteger), + }, + relationships: { + user: relationship(['user_id'], ['public.users.id'], 'one-to-one'), + }, + }); + + const _publicSchema = schema('public', { + users: usersTable, + posts: postsTable, + type_mismatch: typeMismatchTable, + }); + + type _TestSchemas = { + public: typeof _publicSchema; + }; + + void it('returns empty array when all relationships are valid', () => { + type Result = ValidateTableRelationships< + typeof postsTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = Expect>; + }); + + void it('collects errors when relationship has length mismatch', () => { + const _invalidTable = table('invalid', { + columns: { + col1: column('col1', Integer), + col2: column('col2', Integer), + }, + relationships: { + bad_rel: relationship( + ['col1', 'col2'], + ['public.users.id'], + 'one-to-one', + ), + }, + }); + + type Result = ValidateTableRelationships< + typeof _invalidTable, + typeof _publicSchema, + _TestSchemas + >; + + type Expected = TypeValidationResult< + false, + { + table: 'invalid'; + errors: [ + { + relationship: 'bad_rel'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: readonly ['col1', 'col2']; + references: readonly ['public.users.id']; + }, + ]; + }, + ]; + } + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('collects errors when relationship references missing schema', () => { + const _missingSchemaTable = table('missing_schema', { + columns: { + user_id: column('user_id', Integer), + }, + relationships: { + bad_schema_rel: relationship( + ['user_id'], + ['nonexistent.users.id'], + 'one-to-one', + ), + }, + }); + + type Result = ValidateTableRelationships< + typeof _missingSchemaTable, + typeof _publicSchema, + _TestSchemas + >; + + type Expected = TypeValidationResult< + false, + { + table: 'missing_schema'; + errors: [ + { + relationship: 'bad_schema_rel'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.users.id'; + }, + ]; + }, + ]; + } + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('collects errors from multiple invalid relationships', () => { + const _multiErrorTable = table('multi_error', { + columns: { + col1: column('col1', Integer), + col2: column('col2', Integer), + }, + relationships: { + rel1: relationship(['col1', 'col2'], ['public.users.id'], 'one-to-one'), + rel2: relationship(['col1'], ['nonexistent.table.id'], 'one-to-one'), + }, + }); + + type Result = ValidateTableRelationships< + typeof _multiErrorTable, + typeof _publicSchema, + _TestSchemas + >; + + type Expected = TypeValidationResult< + false, + { + table: 'multi_error'; + errors: [ + { + relationship: 'rel1'; + errors: [ + { + errorCode: 'reference_length_mismatch'; + columns: readonly ['col1', 'col2']; + references: readonly ['public.users.id']; + }, + ]; + }, + { + relationship: 'rel2'; + errors: [ + { + errorCode: 'missing_schema'; + reference: 'nonexistent.table.id'; + }, + ]; + }, + ]; + } + >; + + type _Then = [Expect>, Expect>]; + }); + + void it('returns empty array when table has no relationships', () => { + const _noRelTable = table('no_rels', { + columns: { + id: column('id', Integer), + }, + }); + + type Result = ValidateTableRelationships< + typeof _noRelTable, + typeof _publicSchema, + _TestSchemas + >; + + type _Then = Expect>; + }); + + void it('collects errors for type mismatch', () => { + type Result = ValidateTableRelationships< + typeof typeMismatchTable, + typeof _publicSchema, + _TestSchemas + >; + + type Expected = { + valid: false; + error: { + table: 'type_mismatch'; + errors: [ + { + relationship: 'user'; + errors: [ + { + errorCode: 'type_mismatch'; + reference: 'public.users.id'; + referenceType: 'INTEGER'; + columnTypeName: 'BIGINT'; + }, + ]; + }, + ]; + }; + }; + + type _Then = [Expect>, Expect>]; + }); +}); diff --git a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts index deb496c5..81b980c7 100644 --- a/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/components/tableSchemaComponent.ts @@ -26,15 +26,17 @@ export type TableColumns = Record; export type TableSchemaComponent< Columns extends TableColumns = TableColumns, - // eslint-disable-next-line @typescript-eslint/no-empty-object-type - Relationships extends TableRelationships = {}, + TableName extends string = string, + Relationships extends TableRelationships< + keyof Columns & string + > = {} & TableRelationships, > = SchemaComponent< TableURN, Readonly<{ - tableName: string; + tableName: TableName; columns: ReadonlyMap & Columns; primaryKey: TableColumnNames< - TableSchemaComponent + TableSchemaComponent >[]; relationships: Relationships; indexes: ReadonlyMap; @@ -43,11 +45,25 @@ export type TableSchemaComponent< }> >; +export type InferTableSchemaComponentTypes = + T extends TableSchemaComponent< + infer Columns, + infer TableName, + infer Relationships + > + ? [Columns, TableName, Relationships] + : never; + +export type InferTableSchemaComponentColumns< + T extends AnyTableSchemaComponent, +> = InferTableSchemaComponentTypes[0]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -export type AnyTableSchemaComponent = TableSchemaComponent; +export type AnyTableSchemaComponent = TableSchemaComponent; export const tableSchemaComponent = < - Columns extends TableColumns = TableColumns, + const Columns extends TableColumns = TableColumns, + const TableName extends string = string, // eslint-disable-next-line @typescript-eslint/no-empty-object-type const Relationships extends TableRelationships = {}, >({ @@ -57,11 +73,17 @@ export const tableSchemaComponent = < relationships, ...migrationsOrComponents }: { - tableName: string; + tableName: TableName; columns?: Columns; - primaryKey?: TableColumnNames>[]; + primaryKey?: TableColumnNames< + TableSchemaComponent + >[]; relationships?: Relationships; -} & SchemaComponentOptions): TableSchemaComponent & { +} & SchemaComponentOptions): TableSchemaComponent< + Columns, + TableName, + Relationships +> & { relationships: Relationships; } => { columns ??= {} as Columns; @@ -98,7 +120,7 @@ export const tableSchemaComponent = < }, addColumn: (column: AnyColumnSchemaComponent) => base.addComponent(column), addIndex: (index: IndexSchemaComponent) => base.addComponent(index), - } as TableSchemaComponent & { + } as TableSchemaComponent & { relationships: Relationships; }; }; diff --git a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts index 78c9ca80..6d3c76a0 100644 --- a/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.ts @@ -17,6 +17,10 @@ import type { TableSchemaComponent, } from './tableSchemaComponent'; +export type Writable = { + -readonly [P in keyof T]: T[P]; +}; + export type InferColumnType = ColumnType extends ColumnTypeToken< infer _JSType, @@ -39,9 +43,9 @@ export type TableColumnNames = Exclude< keyof ReadonlyMap >; -export type InferTableRow = { +export type InferTableRow = Writable<{ [K in keyof Columns]: TableColumnType; -}; +}>; export type TableRowType = T extends TableSchemaComponent diff --git a/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts b/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts index 100cb294..4692c87d 100644 --- a/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts +++ b/src/packages/dumbo/src/core/schema/components/tableTypesInference.type.spec.ts @@ -9,7 +9,7 @@ import type { TimestamptzToken, VarcharToken, } from '../../sql/tokens/columnTokens'; -import type { Equal, Expect } from '../../testing'; +import type { Equals, Expect } from '../../testing'; import { dumboSchema } from '../dumboSchema'; import type { InferColumnType, @@ -23,32 +23,32 @@ const { Serial, BigSerial, Integer, BigInteger, Varchar, Timestamp, JSONB } = SQL.column.type; // InferColumnValueType - basic types -type _Test1 = Expect, number>>; -type _Test2 = Expect, bigint>>; -type _Test3 = Expect, number>>; -type _Test4 = Expect, bigint>>; -type _Test5 = Expect, string>>; -type _Test6 = Expect, Date>>; -type _Test7 = Expect, Date>>; +type _Test1 = Expect, number>>; +type _Test2 = Expect, bigint>>; +type _Test3 = Expect, number>>; +type _Test4 = Expect, bigint>>; +type _Test5 = Expect, string>>; +type _Test6 = Expect, Date>>; +type _Test7 = Expect, Date>>; // InferColumnValueType - JSONB with custom type type CustomType = { foo: string; bar: number }; type _Test8 = Expect< - Equal>, CustomType> + Equals>, CustomType> >; // InferColumnType - primary key is non-nullable const _idColumn = column('id', Serial, { primaryKey: true }); -type _Test9 = Expect, number>>; +type _Test9 = Expect, number>>; // InferColumnType - notNull is non-nullable const _emailColumn = column('email', Varchar(255), { notNull: true }); -type _Test10 = Expect, string>>; +type _Test10 = Expect, string>>; // InferColumnType - default column is nullable const _nicknameColumn = column('nickname', Varchar(100)); type _Test11 = Expect< - Equal, string | null> + Equals, string | null> >; // InferColumnType - column with default is still nullable @@ -56,27 +56,27 @@ const _createdAtColumn = column('createdAt', Timestamp, { default: SQL.plain(`NOW()`), }); type _Test12 = Expect< - Equal, Date | null> + Equals, Date | null> >; // InferColumnType - unique column is nullable const _usernameColumn = column('username', Varchar(50), { unique: true }); type _Test13 = Expect< - Equal, string | null> + Equals, string | null> >; // InferColumnType - serial without primary key is nullable const _sortOrderColumn = column('sortOrder', Serial); type _Test14 = Expect< - Equal, number | null> + Equals, number | null> >; // InferColumnType - bigint types const _bigIdColumn = column('bigId', BigSerial, { primaryKey: true }); const _nullableBigIntColumn = column('bigValue', BigInteger); -type _Test15 = Expect, bigint>>; +type _Test15 = Expect, bigint>>; type _Test16 = Expect< - Equal, bigint | null> + Equals, bigint | null> >; // InferTableRow - complex table with mixed nullability @@ -93,7 +93,7 @@ const _usersTable = table('users', { }); type UserRow = InferTableRow; type _Test17 = Expect< - Equal< + Equals< UserRow, { id: number; @@ -118,7 +118,7 @@ const _productsTable = table('products', { }); type ProductRow = TableRowType; type _Test18 = Expect< - Equal< + Equals< ProductRow, { id: bigint; @@ -141,7 +141,7 @@ const _strictTable = table('strict', { }); type StrictRow = TableRowType; type _Test19 = Expect< - Equal< + Equals< StrictRow, { id: number; @@ -163,7 +163,7 @@ const _nullableTable = table('nullable', { }); type NullableRow = TableRowType; type _Test20 = Expect< - Equal< + Equals< NullableRow, { id: number; diff --git a/src/packages/dumbo/src/core/schema/dumboFeature.md b/src/packages/dumbo/src/core/schema/dumboFeature.md deleted file mode 100644 index eaccab5b..00000000 --- a/src/packages/dumbo/src/core/schema/dumboFeature.md +++ /dev/null @@ -1,1606 +0,0 @@ -read src/packages/dumbo/src/core/schema/MIGRATION_UNIFICATION_PLAN.md first, then tell me how would you recommend to join it with src/packages/pongo/src/core/schema/index.ts where pongo database could be either database or database per database schema. Also how would that fit Architecture as Code defined as: - -export type EmmettRelationshipType = string; - -export type PortRequirements = Record; - -export type EmmettArchModule< -Requires extends PortRequirements | undefined = undefined, -Exposes extends PortRequirements | undefined = undefined, - -> = { -> name: string; -> } & (Requires extends undefined -> ? Exposes extends undefined - - ? { ports?: undefined } // both undefined: ports optional - : { ports: { exposes: Exposes } } // only Exposes defined - -: Exposes extends undefined -? { ports: { requires: Requires } } // only Requires defined -: { ports: { requires: Requires; exposes: Exposes } }); // both defined - -export type AnyEmmettArchModule = EmmettArchModule; - -export interface EmmettRelationship< -Source extends AnyEmmettArchModule = AnyEmmettArchModule, -Target extends AnyEmmettArchModule = AnyEmmettArchModule, - -> { -> source: Source['name']; -> target: Target['name']; -> type: EmmettRelationshipType; -> description?: string; -> } - -export type EmmettRelationshipsMap< -Source extends AnyEmmettArchModule = AnyEmmettArchModule, -Target extends AnyEmmettArchModule = AnyEmmettArchModule, - -> = Record>; - -export type EmmettComponent< -NestedComponents extends -// eslint-disable-next-line @typescript-eslint/no-explicit-any -Record> | undefined = undefined, -Requires extends PortRequirements | undefined = undefined, -Exposes extends PortRequirements | undefined = undefined, - -> = EmmettArchModule & -> (NestedComponents extends undefined - - ? { components?: undefined } - : { components: NestedComponents }); - -export interface EmmettContainer< -T extends Record = Record, - -> extends EmmettArchModule { -> components?: T; -> } - -export interface EmmettSystem< -T extends Record = Record, - -> extends EmmettArchModule { -> containers?: T; -> } - -// export type EmmettComponentsMap> = { -// [K in keyof T]: EmmettComponent< -// T[K] extends EmmettComponent ? U : unknown -// >; -// }; - -export type EmmettContainersMap> = { -[K in keyof T]: EmmettContainer< -T[K] extends EmmettContainer ? U : Record - -> ; -> }; - -export type EmmettSystemsMap> = { -[K in keyof T]: EmmettSystem< -T[K] extends EmmettSystem ? U : Record - -> ; -> }; - -// const emmettComponent = , 'name'>>( -// name: string, -// config?: T, -// ) => { -// return { name, ...config } satisfies EmmettComponent; -// }; - -export type ComponentsOf = T extends { -components: infer M; -} -? M -: undefined; - -export function emmettComponent< -const Config extends { -// eslint-disable-next-line @typescript-eslint/no-explicit-any -components?: Record>; -ports?: { -requires?: PortRequirements; -exposes?: PortRequirements; -}; -}, - -> ( -> name: string, -> config?: Config, -> ): { -> name: string; -> components: Config extends { components: infer C } ? C : undefined; -> ports: Config extends { ports: infer P } ? P : undefined; -> } { -> return { - - name, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any - components: config?.components as any, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any - ports: config?.ports as any, - -}; -} - -// ({ -// name, -// relationships: config?.relationships ? config.relationships : undefined, -// }) satisfies EmmettComponent as unknown as TReturn; - -function emmettContainer>( -name: string, -components?: T, -): EmmettContainer { -return { -name, -components, -}; -} - -const emmettSystem = >( -name: string, -containers?: T, -): EmmettSystem => ({ -name, -containers, -}); - -const emmettRelationship = < -Source extends AnyEmmettArchModule, -Target extends AnyEmmettArchModule, - -> ( -> source: Source, -> type: EmmettRelationshipType, -> target: Target, -> bundle?: ( - - target: Target['ports']['exposes'], - -) => Partial, -description?: string, -): EmmettRelationship => ({ -source: source.name, -target: target.name, -type, -description, -}); - -type ModuleBuilder = { -relationship: ( -target: Target, -type: EmmettRelationshipType, -description?: string, -) => EmmettRelationship; -}; - -export const moduleBuilder = ( -ctx: Source, -): ModuleBuilder => ({ -relationship: ( -target: Target, -type: EmmettRelationshipType, -description?: string, -) => emmettRelationship(ctx, type, target, undefined, description), -}); - -export const emmettArch = { -system: emmettSystem, -container: emmettContainer, -component: emmettComponent, -relationship: emmettRelationship, -}; - -and - -import { emmettArch } from '.'; - -const { component, container, relationship } = emmettArch; - -const query = -() => -(\_input: Input) => -Promise.resolve({} as Output); - -const getGuestByExternalId = (\_externalId: string): Promise => -Promise.resolve(\_externalId); - -const guests = component('guests', { -components: {}, -ports: { -requires: {}, -exposes: { -queries: { -getGuestByExternalId, -}, -}, -}, -}); - -const pricing = component('pricing'); - -const groupReservations = component('group-reservations'); - -const reservations = component('reservations', { -components: { groupReservations }, -ports: { -requires: { -guests: { -getGuestByExternalId: query(), -}, -}, -exposes: {}, -}, -}); - -const reservationsToGuests = relationship( -reservations, -'provides guest information to', -guests, -({ queries: { getGuestByExternalId } }) => ({ -guests: { -getGuestByExternalId, -}, -}), -); - -const hotelManagement = container('hotel-management', { -guests, -reservations, -pricing, -}); - -Where we could define container and pass dumboSchema to define modules inside e.g. postgresqlContainer (in the c4 model sense), - -Also how could I define also event store having multiple tables and functions, e.g. - -import type { SQLiteConnection } from '../../connection'; -import { -globalTag, -messagesTable, -streamsTable, -subscriptionsTable, -} from './typing'; - -export const sql = (sql: string) => sql; - -export const streamsTableSQL = sql( -`CREATE TABLE IF NOT EXISTS ${streamsTable.name}( - stream_id TEXT NOT NULL, - stream_position BIGINT NOT NULL DEFAULT 0, - partition TEXT NOT NULL DEFAULT '${globalTag}', - stream_type TEXT NOT NULL, - stream_metadata JSONB NOT NULL, - is_archived BOOLEAN NOT NULL DEFAULT FALSE, - PRIMARY KEY (stream_id, partition, is_archived), - UNIQUE (stream_id, partition, is_archived) - );`, -); - -export const messagesTableSQL = sql( -`CREATE TABLE IF NOT EXISTS ${messagesTable.name}( - stream_id TEXT NOT NULL, - stream_position BIGINT NOT NULL, - partition TEXT NOT NULL DEFAULT '${globalTag}', - message_kind CHAR(1) NOT NULL DEFAULT 'E', - message_data JSONB NOT NULL, - message_metadata JSONB NOT NULL, - message_schema_version TEXT NOT NULL, - message_type TEXT NOT NULL, - message_id TEXT NOT NULL, - is_archived BOOLEAN NOT NULL DEFAULT FALSE, - global_position INTEGER PRIMARY KEY, - created DATETIME DEFAULT CURRENT_TIMESTAMP, - UNIQUE (stream_id, stream_position, partition, is_archived) - ); -`, -); - -export const subscriptionsTableSQL = sql( -` CREATE TABLE IF NOT EXISTS ${subscriptionsTable.name}( - subscription_id TEXT NOT NULL, - version INTEGER NOT NULL DEFAULT 1, - partition TEXT NOT NULL DEFAULT '${globalTag}', - last_processed_position BIGINT NOT NULL, - PRIMARY KEY (subscription_id, partition, version) - );`, -); - -export const schemaSQL: string[] = [ -streamsTableSQL, -messagesTableSQL, -subscriptionsTableSQL, -]; - -export const createEventStoreSchema = async ( -db: SQLiteConnection, -): Promise => { -for (const sql of schemaSQL) { -await db.command(sql); -} -}; - -export const appendToStreamSQL = rawSql( -`CREATE OR REPLACE FUNCTION emt_append_to_stream( -v_message_ids text[], -v_messages_data jsonb[], -v_messages_metadata jsonb[], -v_message_schema_versions text[], -v_message_types text[], -v_message_kinds text[], -v_stream_id text, -v_stream_type text, -v_expected_stream_position bigint DEFAULT NULL, -v_partition text DEFAULT emt_sanitize_name('default_partition') -) RETURNS TABLE ( -success boolean, -next_stream_position bigint, -global_positions bigint[], -transaction_id xid8 -) LANGUAGE plpgsql -AS $$ -DECLARE -v_next_stream_position bigint; -v_position bigint; -v_updated_rows int; -v_transaction_id xid8; -v_global_positions bigint[]; -BEGIN -v_transaction_id := pg_current_xact_id(); - - IF v_expected_stream_position IS NULL THEN - SELECT COALESCE( - (SELECT stream_position - FROM ${streamsTable.name} - WHERE stream_id = v_stream_id - AND partition = v_partition - AND is_archived = FALSE - LIMIT 1), - 0 - ) INTO v_expected_stream_position; - END IF; - - v_next_stream_position := v_expected_stream_position + array_upper(v_messages_data, 1); - - IF v_expected_stream_position = 0 THEN - INSERT INTO ${streamsTable.name} - (stream_id, stream_position, partition, stream_type, stream_metadata, is_archived) - VALUES - (v_stream_id, v_next_stream_position, v_partition, v_stream_type, '{}', FALSE); - ELSE - UPDATE ${streamsTable.name} as s - SET stream_position = v_next_stream_position - WHERE stream_id = v_stream_id AND stream_position = v_expected_stream_position AND partition = v_partition AND is_archived = FALSE; - - get diagnostics v_updated_rows = row_count; - - IF v_updated_rows = 0 THEN - RETURN QUERY SELECT FALSE, NULL::bigint, NULL::bigint[], NULL::xid8; - RETURN; - END IF; - END IF; - - WITH ev AS ( - SELECT row_number() OVER () + v_expected_stream_position AS stream_position, - message_data, - message_metadata, - schema_version, - message_id, - message_type, - message_kind - FROM ( - SELECT * - FROM - unnest(v_message_ids, v_messages_data, v_messages_metadata, v_message_schema_versions, v_message_types, v_message_kinds) - AS message(message_id, message_data, message_metadata, schema_version, message_type, message_kind) - ) AS message - ), - all_messages_insert AS ( - INSERT INTO ${messagesTable.name} - (stream_id, stream_position, partition, message_data, message_metadata, message_schema_version, message_type, message_kind, message_id, transaction_id) - SELECT - v_stream_id, ev.stream_position, v_partition, ev.message_data, ev.message_metadata, ev.schema_version, ev.message_type, ev.message_kind, ev.message_id, v_transaction_id - FROM ev - RETURNING global_position - ) - SELECT - array_agg(global_position ORDER BY global_position) INTO v_global_positions - FROM - all_messages_insert; - - RETURN QUERY SELECT TRUE, v_next_stream_position, v_global_positions, v_transaction_id; - -END; - -$$ -; -`, -); - -Potentially in different schemas (e.g. event store in one schema, read models using pongo in others. - -How could this be explicit? would we introduce some databaseFeatureSchemaComponent or databaseSchemaFeatureSchemaComponent? - -How to make it explicit that we're adding feature that can have multiple types of components but also be able to locate it in the specific schema or database? - -If I'm not clear enough, please ask for more. Ultrathink -[Show more] -I'll read the migration plan first to understand the context, then analyze how to integrate it with Pongo's schema system and the Architecture as Code concept. -Read MIGRATION_UNIFICATION_PLAN.md, index.ts -Now let me understand the question better by looking at some examples and clarifying a few points. I can see: -Dumbo's migration unification plan - A comprehensive system for managing database schemas with migrations, snapshots, and TypeScript definitions -Pongo's schema system - A higher-level abstraction focused on MongoDB-like collections within databases -Emmett Architecture as Code - A C4-model-inspired system for defining components, containers, and systems with ports and relationships -The user is asking about how to integrate these three concepts, specifically: -How to connect Dumbo's schema components with Pongo's database/collection abstractions -How to fit this into the Emmett Architecture as Code model -How to make it explicit when features span multiple schemas or databases -Dealing with the fact that Pongo can be "database or database per database schema" -Let me search for more context about event stores and how they're currently used: -Globpattern: "**/event*store*schema*.ts" -⎿ -Found 1 file -Grep "eventStoreSchema|event.*store.*component" -⎿ -1 line of output -Grep "databaseFeature|schemaFeature" -⎿ -1 line of output -Let me look for how dumbo schema components are currently structured: -Read dumboSchema.ts -Globpattern: "**/examples/**/*.ts" -⎿ -Found 1 file -Now I have a comprehensive understanding. Let me analyze the integration challenge and provide recommendations. The question involves several complex integration points: -Dumbo schema components - Low-level SQL schema definition with migrations -Pongo schema - MongoDB-like abstraction over SQL databases -Emmett Architecture as Code - C4-model components with ports and relationships -Physical deployment - Different database schemas or databases -Let me formulate my recommendations: -Claude’s Plan -Integration Plan: Dumbo + Pongo + Emmett Architecture as Code -Context Analysis -After reviewing the migration plan and existing code, I see three layers that need integration: -Dumbo Layer: Low-level database schema with SQL migrations (database → schema → table → column) -Pongo Layer: MongoDB-like abstraction (client → db → collection) -Emmett Layer: C4-model architecture components with ports/relationships -Key Challenge: "Pongo can be database OR database-per-schema" -This is the critical mapping issue: -PostgreSQL mode: One database, multiple schemas → Each Pongo DB maps to a PostgreSQL schema -SQLite mode: One database, no schema support → Each Pongo DB is just a logical grouping -Multi-tenant mode: Multiple databases → Each Pongo DB maps to a separate physical database -Recommended Architecture -1. Introduce databaseFeatureComponent -This bridges Dumbo's physical schema and Emmett's logical components: -export type DatabaseFeatureComponent< -NestedComponents extends Record> | undefined = undefined, -Requires extends PortRequirements | undefined = undefined, -Exposes extends PortRequirements | undefined = undefined, -> = EmmettComponent & { -// Physical database mapping -deployment: { - mode: 'database' | 'schema' | 'collection-prefix'; - database?: string; // Physical database name - schema?: string; // PostgreSQL schema name - prefix?: string; // For collection prefixing in shared DB -}; - -// Dumbo schema definition -dumboSchema: DatabaseSchemaComponent | DatabaseSchemaSchemaComponent; - -// Pongo schema definition -pongoSchema: PongoDbSchema; -}; -2. Event Store Example Integration -const eventStoreFeature = emmettArch.component('event-store', { -components: { - streams: emmettArch.component('streams-table'), - messages: emmettArch.component('messages-table'), - subscriptions: emmettArch.component('subscriptions-table'), - appendToStream: emmettArch.component('append-to-stream-function'), -}, -ports: { - exposes: { - commands: { - appendToStream: (/* ... */) => Promise.resolve(), - }, - queries: { - readStream: (/* ... */) => Promise.resolve([]), - }, - }, -}, -deployment: { - mode: 'schema', - database: 'main_db', - schema: 'event_store', -}, -dumboSchema: dumboDatabaseSchema('event_store', { - streams: dumboTable('streams', { /* ... */ }), - messages: dumboTable('messages', { /* ... */ }), - subscriptions: dumboTable('subscriptions', { /* ... */ }), -}), -pongoSchema: pongoSchema.db('event_store', { - streams: pongoSchema.collection('streams'), - messages: pongoSchema.collection('messages'), - subscriptions: pongoSchema.collection('subscriptions'), -}), -}); -3. Multi-Schema Container -For your hotel management example with separate schemas: -const hotelManagementContainer = emmettArch.container('hotel-management', { -// Event store in dedicated schema -eventStore: databaseFeatureComponent('event-store', { - deployment: { mode: 'schema', schema: 'event_store' }, - dumboSchema: eventStoreDumboSchema, - pongoSchema: eventStorePongoSchema, -}), - -// Read models in another schema -readModels: databaseFeatureComponent('read-models', { - deployment: { mode: 'schema', schema: 'read_models' }, - dumboSchema: readModelsDumboSchema, - pongoSchema: readModelsPongoSchema, - components: { - guests: emmettArch.component('guests'), - reservations: emmettArch.component('reservations'), - }, -}), -}); -4. Type-Safe Schema Navigation -Add helper to navigate from Emmett component to its schemas: -export const getComponentSchemas = ( -component: T -) => ({ -dumbo: component.dumboSchema, -pongo: component.pongoSchema, -deployment: component.deployment, - -// Get fully qualified table names -tables: () => { - const { mode, database, schema, prefix } = component.deployment; - // Return table references with correct qualifiers -}, -}); -5. Migration Integration -The orchestrator should understand deployment modes: -export class MultiTenantMigrationOrchestrator { -async deployFeature(feature: DatabaseFeatureComponent) { - const { mode, database, schema } = feature.deployment; - - switch (mode) { - case 'database': - // Deploy to separate database - await this.migrateDatabase(database!, feature.dumboSchema); - break; - case 'schema': - // Deploy to schema within database - await this.migrateSchema(database!, schema!, feature.dumboSchema); - break; - case 'collection-prefix': - // Just prefix collections, no migration needed - break; - } -} -} -Questions for Clarification -Naming: Should we call it databaseFeatureComponent or dataStoreComponent or persistenceComponent? -Nested features: If a feature has multiple schemas (e.g., event_store + projections), should each be a separate component or nested? -Cross-schema relationships: How should we express that read models depend on event store when they're in different schemas? -Collection prefixing: Should we support a "prefix" mode where multiple features share a schema but use prefixed collections? -Function/procedure components: Should stored procedures be first-class components or just part of table components? -Next Steps -Define DatabaseFeatureComponent type with deployment metadata -Extend Emmett arch builder with .databaseFeature() method -Create mapping helpers between Dumbo/Pongo/Emmett schemas -Update migration orchestrator to handle deployment modes -Add examples showing event store + read models in separate schemas -Stayed in plan mode -No, event store schema component should define all on their own. Potentially it can take pongoSchema or dumboSchema for inline projections. Read https://event-driven.io/en/emmett_projections_testing/ - -PongoSchema defines collections like mongo db, not eventstroe read: https://event-driven.io/en/pongo_strongly_typed_client/ - -I'd likesomeone to need to define all dumboSchema, pongo Schema etc. I'd like to be able to either wrap dumbo schema, or be able to pass those features to tell, e.g. like we have tables then including event store. or pongo collection. - -The obvious option is to include dumbodatabase schema into event store and pongoschema and build it with wrappers, but I'd like to avoid it if possible to just compose it as other schemaComponents -Ah, I understand much better now! Let me re-read those articles and rethink this. -Web Fetchhttps://event-driven.io/en/emmett_projections_testing/ -Fetched from https://event-driven.io/en/emmett_projections_testing/ -Web Fetchhttps://event-driven.io/en/pongo_strongly_typed_client/ -Fetched from https://event-driven.io/en/pongo_strongly_typed_client/ -Let me search for event store and projection examples in the codebase: -Grep "inline.*projection|projection.*inline" -⎿ -1 line of output -Grep "eventStore|event-store|EventStore" -⎿ -2 lines of output -Globpattern: "**/emmett**/projection*.ts" -⎿ -Found 1 file -Grep "class.*EventStore|function.*eventStore|const.*eventStore" -⎿ -1 line of output -Globpattern: "**/eventStore*.ts" -⎿ -Found 1 file -Globpattern: "**/event-store*.ts" -⎿ -Found 1 file -Let me look at the schema component structure to understand how components compose: -Read schemaComponent.ts, tableSchemaComponent.ts -If you don't know something then ask, don't hallucinate. I provided you a lot. Event Store is in emmett, not in this repo, that's why I've sent you code samples and articles... -If you need some more definition please ask. -Perfect! Now I understand the pattern. Let me think through what you're asking: Your key insights: -Event store should define its own schema components (tables, functions, etc.) -Pongo schema is for MongoDB-like collections, NOT for event store -You want to compose features like event store AS schema components, not wrap them -You want to avoid having to define dumbo/pongo schemas separately inside each feature -The pattern I see: -TableSchemaComponent extends SchemaComponent with table-specific data -It can be added to a DatabaseSchemaSchemaComponent as a component -You want something similar for "event store" or "pongo collections" -Let me propose a solution based on this understanding: -Proposed Solution: Feature Schema Components -1. EventStoreSchemaComponent (Dumbo side) -export type EventStoreURNType = 'sc:dumbo:event_store'; -export type EventStoreURN = `${EventStoreURNType}:${string}`; - -export type EventStoreSchemaComponent = SchemaComponent< -EventStoreURN, -Readonly<{ - eventStoreName: string; - tables: ReadonlyMap; - functions: ReadonlyMap; -}> ->; - -export const eventStoreSchemaComponent = ({ -eventStoreName, -...options -}: { -eventStoreName: string; -} & SchemaComponentOptions): EventStoreSchemaComponent => { -// Define the event store tables -const streamsTable = tableSchemaComponent({ - tableName: 'streams', - migrations: [/* SQL for streams table */] -}); - -const messagesTable = tableSchemaComponent({ - tableName: 'messages', - migrations: [/* SQL for messages table */] -}); - -const subscriptionsTable = tableSchemaComponent({ - tableName: 'subscriptions', - migrations: [/* SQL for subscriptions table */] -}); - -const appendFunction = functionSchemaComponent({ - functionName: 'emt_append_to_stream', - migrations: [/* SQL for function */] -}); - - const base = schemaComponent( - `sc:dumbo:event_store:${eventStoreName}` as EventStoreURN, - { - migrations: options.migrations ?? [], - components: [ - streamsTable, - messagesTable, - subscriptionsTable, - appendFunction, - ...(options.components ?? []), - ], - } -); - - return { - ...base, - eventStoreName, - get tables() { - return mapSchemaComponentsOfType( - base.components, - 'sc:dumbo:table', - (c) => c.tableName, - ); - }, - get functions() { - return mapSchemaComponentsOfType( - base.components, - 'sc:dumbo:function', - (c) => c.functionName, - ); - }, -}; -}; -2. PongoCollectionSetSchemaComponent (Pongo side) -For Pongo, you'd have a similar component that represents a set of collections: -export type PongoCollectionSetURNType = 'sc:pongo:collection_set'; -export type PongoCollectionSetURN = `${PongoCollectionSetURNType}:${string}`; - -export type PongoCollectionSetSchemaComponent = SchemaComponent< -PongoCollectionSetURN, -Readonly<{ - name: string; - pongoSchema: PongoDbSchema; - collections: ReadonlyMap; -}> ->; - -export const pongoCollectionSetSchemaComponent = ({ -name, -collections, -...options -}: { -name: string; -collections: Record; -} & SchemaComponentOptions): PongoCollectionSetSchemaComponent => { -const pongoDbSchema = pongoSchema.db(collections); - -const base = schemaComponent( - `sc:pongo:collection_set:${name}` as PongoCollectionSetURN, - { - migrations: options.migrations ?? [], - components: options.components ?? [], - } -); - - return { - ...base, - name, - pongoSchema: pongoDbSchema, - get collections() { - // Could map from pongoSchema.collections - return new Map(); - }, -}; -}; -3. Usage: Composing into DatabaseSchema -// Define event store as a schema component -const eventStore = eventStoreSchemaComponent({ -eventStoreName: 'default', -}); - -// Define read models as Pongo collections wrapped in dumbo tables -const readModelsSchema = dumboDatabaseSchema('read_models', { -guests: dumboTable('guests', { - columns: { - id: dumboColumn('id'), - name: dumboColumn('name'), - } -}), -reservations: dumboTable('reservations', { - columns: { - id: dumboColumn('id'), - guestId: dumboColumn('guest_id'), - } -}), -}); - -// Compose them together -const hotelDatabase = dumboDatabase('hotel', { -event_store: dumboDatabaseSchema('event_store', { - // Add event store as components! Tables are automatically included -}).addComponent(eventStore), // <-- Event store adds its tables - -read_models: readModelsSchema, -}); -Wait, that's still awkward. Let me reconsider... -Better Approach: Event Store IS a DatabaseSchemaSchemaComponent -export const eventStoreSchema = ( -schemaName: string = 'event_store', -options?: SchemaComponentOptions -): DatabaseSchemaSchemaComponent => { -return dumboDatabaseSchema(schemaName, { - streams: dumboTable('streams', { - migrations: [sqlMigration('create_streams', [streamsTableSQL])], - }), - messages: dumboTable('messages', { - migrations: [sqlMigration('create_messages', [messagesTableSQL])], - }), - subscriptions: dumboTable('subscriptions', { - migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], - }), -}, options); -}; - -// Then you can use it directly: -const hotelDatabase = dumboDatabase('hotel', { -event_store: eventStoreSchema('event_store'), -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), -}), -}); -But you also want inline projections... Let me think about that: -With Inline Projections -export const eventStoreSchema = ({ -schemaName = 'event_store', -inlineProjections = {}, -...options -}: { -schemaName?: string; -inlineProjections?: Record; -} & SchemaComponentOptions = {}): DatabaseSchemaSchemaComponent => { -const coreEventStoreTables = { - streams: dumboTable('streams', { - migrations: [sqlMigration('create_streams', [streamsTableSQL])], - }), - messages: dumboTable('messages', { - migrations: [sqlMigration('create_messages', [messagesTableSQL])], - }), - subscriptions: dumboTable('subscriptions', { - migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], - }), -}; - -// Convert inline projections to dumbo tables if needed -const projectionTables: Record = {}; -for (const [name, projection] of Object.entries(inlineProjections)) { - if ('tableName' in projection) { - // It's already a TableSchemaComponent - projectionTables[name] = projection; - } else { - // It's a PongoCollectionSchema, create a table for it - projectionTables[name] = dumboTable(projection.name, {}); - } -} - -return dumboDatabaseSchema(schemaName, { - ...coreEventStoreTables, - ...projectionTables, -}, options); -}; - -// Usage: -const hotelDatabase = dumboDatabase('hotel', { -event_store: eventStoreSchema({ - schemaName: 'event_store', - inlineProjections: { - guest_summary: pongoSchema.collection('guest_summary'), - reservation_summary: dumboTable('reservation_summary', { /* ... */ }), - }, -}), -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), -}), -}); -For Emmett Architecture Integration -Now, how does this relate to Emmett components? -const eventStoreComponent = emmettArch.component('event-store', { -ports: { - exposes: { - commands: { - appendToStream: (/* ... */) => Promise.resolve(), - }, - }, -}, -}); - -const guestsComponent = emmettArch.component('guests', { -ports: { - exposes: { - queries: { - getGuestByExternalId: query(), - }, - }, -}, -}); - -// The container ties Emmett components to database schemas -const hotelManagementContainer = emmettArch.container('hotel-management', { -// Emmett components -eventStore: eventStoreComponent, -guests: guestsComponent, -reservations: reservationsComponent, -}); - -// Separately, define the database schema -const hotelDatabaseSchema = dumboDatabase('hotel', { -event_store: eventStoreSchema({ schemaName: 'event_store' }), -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), -}), -}); -But how do we link them? -Key Questions: -Should Emmett components contain schema definitions? Or should they be separate? -Should we have a mapping layer that says "event-store component uses event_store schema"? -Do you want deployment flexibility where the same Emmett component can be deployed to different schemas/databases? -For the "explicit location" problem: Do you want something like: -// Query that knows it's in event_store schema -const streams = database.schema('event_store').table('streams'); - -// Or from Emmett component perspective -const component = container.components.eventStore; -const schema = getSchemaFor(component); // Returns event_store schema -Let me ask some clarifying questions to nail down the design: Clarifying Questions: -Separation of concerns: Should Emmett components (business logic) be completely separate from dumbo schemas (persistence), or should they reference each other? -Event store as primitive: Should eventStoreSchema() be a first-class schema component type (like dumboTable), or just a factory function that creates a DatabaseSchemaSchemaComponent? -Inline projections: When you say "event store can take pongoSchema for inline projections", do you mean: -a) Event store schema should accept PongoCollectionSchema and convert them to tables -b) Event store runtime should support Pongo query API on projection tables -c) Both? -Multi-schema deployment: If you have event_store in one schema and read_models in another, how should the Emmett component know which schema to query? Should there be a deployment config that maps components to schemas? -Function/Stored Procedures: Should SQL functions (like emt_append_to_stream) be first-class schema components, or just part of table/schema migrations? -Please clarify these points so I can provide the right solution! -You're absolutely right - I apologize for making assumptions. Let me ask clarifying questions based on what you've actually provided: -Clarifying Questions -Based on the code samples you provided and the migration plan, I need to understand: -1. Event Store Structure -From your SQL example, I can see the event store has: -Tables: streams, messages, subscriptions -Functions: emt_append_to_stream (PostgreSQL stored procedure) -These are defined via raw SQL (streamsTableSQL, messagesTableSQL, etc.) -Question: Should event store be represented as: -Option A: A factory function that returns a DatabaseSchemaSchemaComponent with all tables pre-configured? -const eventStore = eventStoreSchema('event_store'); // Returns DatabaseSchemaSchemaComponent -Option B: A new schema component type (like EventStoreSchemaComponent extends SchemaComponent)? -Option C: Just a collection of table components that you manually compose? -2. Inline Projections -You mentioned "event store can take pongoSchema or dumboSchema for inline projections" Question: What does this mean in practice? -Do inline projections create additional tables in the same schema as the event store? -Should I be able to do something like: -const eventStore = eventStoreSchema('event_store', { -inlineProjections: { - guestSummary: pongoSchema.collection('guest_summary'), - // This creates a 'guest_summary' table in the event_store schema? -} -}); -3. Pongo vs Dumbo Relationship -From pongo/src/core/schema/index.ts: -Pongo schema defines client -> db -> collections (MongoDB-like) -It's a logical abstraction over the physical database -From the migration plan: -Dumbo schema defines database -> schema -> table -> column (SQL) -It's the physical structure with migrations -Question: How should these relate? -Can a single Pongo db map to multiple Dumbo schemas? -If I have event_store in one schema and read_models in another, do I create: -Two separate Pongo dbs? -One Pongo db that somehow spans both schemas? -4. Stored Procedures / Functions -Your emt_append_to_stream example is a PostgreSQL function. Question: Should functions be: -Option A: Part of table migrations (stored in table component)? -Option B: A new FunctionSchemaComponent type (like TableSchemaComponent)? -Option C: Part of schema-level migrations (stored in DatabaseSchemaSchemaComponent)? -5. Emmett Component Integration -You showed Emmett Architecture as Code with components, containers, ports. Question: What's the relationship between: -Emmett component (business logic with ports) -Dumbo DatabaseSchemaSchemaComponent (physical schema) -Pongo PongoDbSchema (logical collections) -Should they: -Option A: Be completely separate (define them independently, link via configuration)? -Option B: Have explicit references (Emmett component has .schema property)? -Option C: Use a mapping/registry pattern? -6. Multiple Schemas in One Database -Your question mentions "pongo database could be either database or database per database schema" Question: Can you explain this with a concrete example? -// Scenario 1: PostgreSQL with multiple schemas -const db = pongoClient.db('hotel'); // Maps to PostgreSQL database 'hotel' -// This database has schemas: event_store, read_models, analytics - -// Scenario 2: SQLite (no schemas) -const db = pongoClient.db('hotel'); // Maps to SQLite database 'hotel' -// No schema concept, all tables in one namespace - -// How should the schema definition work for both? -7. "Explicit Location" Requirement -You said: "How to make it explicit that we're adding feature that can have multiple types of components but also be able to locate it in the specific schema or database?" Question: What does "locate it" mean? -At definition time: "This feature's tables go in schema X"? -At runtime: "Query this component and know which schema it's in"? -At architecture documentation time: "Show which schema each component uses"? -8. Composition Pattern -You said: "I'd like to just compose it as other schemaComponents" Question: What does the ideal composition look like to you? -// Example A: Flat composition -const hotelDb = dumboDatabase('hotel', { -event_store: eventStoreSchema(), -read_models: readModelsSchema(), -}); - -// Example B: Nested composition -const hotelDb = dumboDatabase('hotel') -.addSchema(eventStoreSchema()) -.addSchema(readModelsSchema()); - -// Example C: Component-based -const hotelDb = dumboDatabase('hotel', { -features: { - eventStore: eventStoreFeature, // This internally creates schemas - readModels: readModelsFeature, -} -}); -Please clarify these points and I'll design the right solution based on what you actually need, not assumptions! -Thank you for your patience. Let me ask the essential questions I need to understand your requirements: -Essential Questions -1. Event Store Package Location & API -You mentioned "Event Store is in emmett, not in this repo" and provided SQL samples. Question: In the Emmett package, does the event store: -Export a function/class that creates the schema definition? -Provide just the SQL strings (like your streamsTableSQL, messagesTableSQL examples)? -Provide a complete schema component I can import and use? -What I need to know: If I want to use event store in this codebase, what exactly do I import from Emmett and how do I turn it into a Dumbo schema component? -2. The Core Integration Problem -You want to avoid wrapping dumbo/pongo schemas inside feature definitions, but still compose them. Question: Starting from your event store SQL example with streams, messages, subscriptions tables and the emt_append_to_stream function - if I want to: -Use this as a Dumbo schema component (for migrations) -Also expose it as a Pongo schema (for querying) -Include it in an Emmett architecture component (for documentation/ports) -Deploy it to a specific PostgreSQL schema (e.g., event_store schema) -What should the code look like? Can you show me a pseudocode example of how you envision defining this? -3. Inline Projections Clarification -You said event store "can take pongoSchema or dumboSchema for inline projections" Question: When you define an inline projection: -// Pseudocode - is this the right direction? -eventStore({ -inlineProjections: { - guestSummary: ??? // What goes here? -} -}) -Does guestSummary become a table in the same PostgreSQL schema as streams/messages? -Is it a Pongo collection definition that gets converted to a table? -Is it a Dumbo table definition? -Something else? -4. Physical Deployment Mapping -Regarding "pongo database could be either database or database per database schema": Question: In these two scenarios, how should the schemas be defined? Scenario A - PostgreSQL (one DB, multiple schemas): --- Physical structure: --- Database: hotel_db --- Schema: event_store (tables: streams, messages, subscriptions) --- Schema: read_models (tables: guests, reservations) -Scenario B - SQLite (one DB, no schema support): --- Physical structure: --- Database: hotel.db --- Tables: streams, messages, subscriptions, guests, reservations -Should the Dumbo/Pongo schema definition be: -The same for both, with deployment config handling the difference? -Different for each database type? -Something else? -5. Function/Stored Procedure Components -Your emt_append_to_stream example is a PostgreSQL function. Question: Should I create a new FunctionSchemaComponent type (similar to TableSchemaComponent), or should functions just be SQL migrations attached to schema/table components? If you want FunctionSchemaComponent, should it be: -const appendFunction = dumboFunction('emt_append_to_stream', { -migrations: [sqlMigration('create_append', [appendToStreamSQL])], -}); - -// Then added to schema -const eventStoreSchema = dumboDatabaseSchema('event_store', { -tables: { streams, messages, subscriptions }, -functions: { appendToStream: appendFunction }, // ??? -}); -6. Emmett Component Connection -Looking at your Emmett Architecture example with emmettArch.component(), emmettArch.container(): Question: Should Emmett components: -Option A: Have no knowledge of Dumbo/Pongo schemas (purely architectural documentation)? -Option B: Reference which schema they use (e.g., eventStoreComponent.schema = eventStoreDumboSchema)? -Option C: Be created FROM schema definitions (e.g., emmettArch.component.fromSchema(eventStoreDumboSchema))? -7. Concrete Use Case -Can you provide a concrete example of what you want to build? For instance: -// I want to define: -// 1. Event store in its own PostgreSQL schema -// 2. Guest read model in another schema -// 3. Both accessible via Pongo API -// 4. Both documented in Emmett architecture -// 5. The event store should have an inline projection for guest summaries - -// What should this code look like? -const ??? = ??? -If you can show me the desired API/syntax you want to write, I can design the implementation to support it. -Please answer whichever questions are relevant to clarify your vision. I want to understand what you're trying to achieve before proposing a solution. -I don't like thaat as some just want to keep event store in the same schema as read model. - -The idea in general is to be able to also run migration together for database that contain both event store and read modls. And yes, by default I don't want user to know internal structure of event store, same for pongo collections. they should be opaque, but in the future give some customisation. -Ah! Now I understand much better. Let me clarify what I'm hearing: -What I Understand Now -Flexibility in deployment: Event store and read models can be in: -Same schema (simpler deployments) -Different schemas (separation of concerns) -User's choice, not forced -Opaque internals: Users shouldn't need to know that event store has streams, messages, subscriptions tables - that's an implementation detail -Run migrations together: Whether event store is in the same schema or different, migrations should run together as one cohesive database setup -Future customization: Later, allow power users to customize event store internals, but not required -Key Question: What Does the API Look Like? -Let me propose what I think you want and you tell me if I'm on the right track: -Scenario 1: Event Store + Read Models in SAME schema -// User just wants everything in 'public' schema -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - // Event store tables are added transparently - ...eventStoreSchemaComponents(), // Returns { streams: table, messages: table, ... } - - // User's read model tables - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), -}) -}); -Scenario 2: Event Store + Read Models in DIFFERENT schemas -// User wants separation -const hotelDb = dumboDatabase('hotel', { -event_store: eventStoreSchema('event_store'), // Opaque, returns DatabaseSchemaSchemaComponent - -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), -}) -}); -Scenario 3: With Inline Projections -const hotelDb = dumboDatabase('hotel', { -event_store: eventStoreSchema('event_store', { - // Inline projections become tables in the SAME schema as event store - inlineProjections: { - guest_summary: pongoSchema.collection('guest_summary'), // Creates guest_summary table - } -}), - -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { /* ... */ }), -}) -}); -Critical Questions: -Q1: Event Store Schema Component API -Should eventStoreSchema() return a DatabaseSchemaSchemaComponent that contains all the tables internally? -// Option A: Opaque schema component -export const eventStoreSchema = ( -schemaName: string, -options?: { inlineProjections?: Record } -): DatabaseSchemaSchemaComponent => { -// Internally creates streams, messages, subscriptions tables -// User doesn't see them -}; - -// Option B: Spread/merge pattern -export const eventStoreSchemaComponents = (): Record => ({ -streams: dumboTable('streams', { migrations: [streamsSQL] }), -messages: dumboTable('messages', { migrations: [messagesSQL] }), -subscriptions: dumboTable('subscriptions', { migrations: [subscriptionsSQL] }), -}); -Which pattern do you prefer? -Q2: Pongo Schema Relationship -If event store and read models are in the same schema, how does Pongo schema map? -// If both in 'public' schema: -const pongoDb = pongoSchema.db('hotel', { -// Do we expose event store collections? -streams: pongoSchema.collection('streams'), // ??? -messages: pongoSchema.collection('messages'), // ??? - -// Or only read models? -guests: pongoSchema.collection('guests'), -reservations: pongoSchema.collection('reservations'), -}); -Should event store tables be exposed via Pongo, or hidden? -Q3: Migration Composition -When running migrations, should event store migrations: -Be part of the normal migration flow (just more tables)? -Run separately/first (event store setup, then read models)? -Be configurable? -Q4: Stored Functions -The emt_append_to_stream PostgreSQL function - should it be: -// Option A: Part of schema migrations -const eventStoreSchema = (schemaName: string) => -dumboDatabaseSchema(schemaName, { - streams: dumboTable('streams', { /* ... */ }), - messages: dumboTable('messages', { /* ... */ }), -}, { - migrations: [ - sqlMigration('create_functions', [appendToStreamSQL]) - ] -}); - -// Option B: First-class function component -const eventStoreSchema = (schemaName: string) => -dumboDatabaseSchema(schemaName, { - streams: dumboTable('streams', { /* ... */ }), - appendToStream: dumboFunction('emt_append_to_stream', { /* ... */ }), -}); -Which approach? -Please tell me: -Which API scenarios (1, 2, 3 above) match what you want? -Which of the options for each question is closer to your vision? -Any corrections to my understanding? -This will help me propose the right design! -I don't like this ...eventStoreSchemaComponents(), - I'd prefer to just use eventStoreSchema(), which could take also inline projection with references to pongo collection, still, pongo collection could be also define inside schema just like we do for table, and pongoDatabase both as database and schema. - -We should be able to make eventStoreSchema and pongo related shcemas to extend some base feature schema component, and then detect it, right or not? -YES! Now I get it! Let me confirm my understanding: -What You Want -1. Event Store as Opaque Component -// User just uses it, doesn't see internal tables -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), // NOT spreading, just add it as a component - guests: dumboTable('guests', { /* ... */ }), - reservations: dumboTable('reservations', { /* ... */ }), -}) -}); -2. Feature Schema Component Pattern -// Base type that event store, pongo collections, etc. extend -export type FeatureSchemaComponent = SchemaComponent< -string, -{ - featureType: 'event_store' | 'pongo_collections' | 'custom'; -} ->; - -// Event store extends this -export type EventStoreSchemaComponent = SchemaComponent< -'sc:dumbo:event_store', -{ - featureType: 'event_store'; - eventStoreName: string; - // Internal: tables, functions - user doesn't see - internalTables: Map; -} ->; - -// Pongo collection set extends this -export type PongoCollectionSetSchemaComponent = SchemaComponent< -'sc:dumbo:pongo_collections', -{ - featureType: 'pongo_collections'; - pongoSchema: PongoDbSchema; -} ->; -3. Detection & Composition -// When adding to schema, detect what type it is -dumboDatabaseSchema('public', { -// This is detected as EventStoreSchemaComponent -eventStore: eventStoreSchema(), - -// This is detected as TableSchemaComponent -guests: dumboTable('guests', { /* ... */ }), - -// This is detected as PongoCollectionSetSchemaComponent -readModels: pongoCollectionsSchema({ - guests: pongoSchema.collection('guests'), - reservations: pongoSchema.collection('reservations'), -}), -}); -4. Inline Projections with Pongo Collections -const hotelDb = dumboDatabase('hotel', { -event_store: dumboDatabaseSchema('event_store', { - // Event store with inline projections - eventStore: eventStoreSchema({ - inlineProjections: { - // Reference to pongo collection - creates table automatically - guestSummary: pongoSchema.collection('guest_summary'), - } - }), -}), - -read_models: dumboDatabaseSchema('read_models', { - // Or define pongo collections directly in schema - readModels: pongoCollectionsSchema({ - guests: pongoSchema.collection('guests'), - reservations: pongoSchema.collection('reservations'), - }), -}), -}); -Key Questions to Confirm: -Q1: Feature Schema Component Base -Should there be a common base or marker for "feature components" that: -Contain multiple internal components (tables, functions, etc.) -Can be detected via isSchemaComponentOfType(comp, 'feature') or similar -Are opaque (internal structure hidden from user) -export const isFeatureSchemaComponent = (comp: AnySchemaComponent): boolean => { -return comp.schemaComponentKey.startsWith('sc:dumbo:feature:'); -}; -Q2: Pongo Collections as Schema Component -Should pongoCollectionsSchema() create tables automatically? -// This creates: -// - A schema component -// - Dumbo tables for each collection (for migrations) -// - Pongo schema for querying -const readModels = pongoCollectionsSchema({ -guests: pongoSchema.collection('guests'), -reservations: pongoSchema.collection('reservations'), -}); - -// Internally generates: -// - dumboTable('guests', {}) -// - dumboTable('reservations', {}) -// - pongoSchema.db({ guests, reservations }) -Is this correct? -Q3: Same Schema Flexibility -All of these should work, right? -// A: Everything in one schema -dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), - guests: dumboTable('guests', {}), -}) -}); - -// B: Separated schemas -dumboDatabase('hotel', { -event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), -}), -read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', {}), -}) -}); - -// C: Event store + pongo collections in same schema -dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), - readModels: pongoCollectionsSchema({ - guests: pongoSchema.collection('guests'), - }), -}) -}); -Q4: Detection in Migration Orchestrator -When collecting migrations, detect feature components: -// In migration orchestrator -for (const component of schema.components.values()) { -if (isEventStoreComponent(component)) { - // Handle event store migrations specially if needed -} else if (isPongoCollectionsComponent(component)) { - // Generate table migrations from pongo collections -} else if (isTableComponent(component)) { - // Standard table migration -} -} -Is this the right approach? -Am I understanding correctly now? The key insight is: -Feature components (event store, pongo collections) are first-class schema components -They compose like tables do -They're detected by type checking -They encapsulate internal structure -Should I proceed with this design? -For user feature should be opaque, they don't need to know how sausages are made, still, they should be visible later in generated typed system and added to the dumbo schema definition so they're strongly typed and also that migrations works fine. - -Ad q3. - -A looks weird as it suggests that eventStore is table, which is not. -For user feature should be opaque, they don't need to know how sausages are made, still, they should be visible later in generated typed system and added to the dumbo schema definition so they're strongly typed and also that migrations works fine. - -Ad q3. - -A looks weird as it suggests that eventStore is table, which is not. -Perfect! This clarifies a lot. Let me refine my understanding: -Key Insights -1. Feature Components are NOT Tables -You're right - in scenario A, putting eventStore: eventStoreSchema() alongside tables is confusing because it's not a table. Better API: -// Option A-revised: Feature at schema level -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - features: { - eventStore: eventStoreSchema(), - }, - tables: { - guests: dumboTable('guests', {}), - reservations: dumboTable('reservations', {}), - } -}) -}); - -// OR Option A-alternative: Feature creates its own implicit schema content -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public') - .withFeature(eventStoreSchema()) - .withTable(dumboTable('guests', {})) -}); - -// OR Option A-simpler: Features and tables mixed but clearly typed -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), // Type system knows this is FeatureSchemaComponent - guests: dumboTable('guests', {}), // Type system knows this is TableSchemaComponent -}) -}); -Question: Which API feels right to you? I'm leaning toward the last one (A-simpler) because: -Type system prevents confusion -Simpler API -Components compose uniformly -2. Strongly Typed Generated System -You want the generated types to expose the feature's tables: -// After running schema generation from: -const hotelDb = dumboDatabase('hotel', { -public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), - guests: dumboTable('guests', {}), -}) -}); - -// Generated types should include: -export type HotelSchema = { -public: { - // Event store tables exposed (even though user didn't define them) - streams: { stream_id: string, stream_position: number, ... }, - messages: { stream_id: string, message_data: unknown, ... }, - subscriptions: { subscription_id: string, ... }, - - // User's tables - guests: { id: string, name: string, ... }, -} -}; -Question: Should the generated types expose event store internals, or keep them hidden? -3. Pongo Schema Generation -For Pongo collections defined via pongoCollectionsSchema(): -const hotelDb = dumboDatabase('hotel', { -read_models: dumboDatabaseSchema('read_models', { - collections: pongoCollectionsSchema({ - guests: pongoSchema.collection('guests'), - reservations: pongoSchema.collection('reservations'), - }), -}) -}); - -// Should generate: -// 1. Dumbo tables: guests, reservations -// 2. Pongo schema: PongoDbSchema with those collections -// 3. Migrations: CREATE TABLE for each collection -Proposed Architecture -Feature Schema Component Base -// Marker interface -export type FeatureSchemaComponent< -URN extends string = string, -AdditionalData extends Record = Record -> = SchemaComponent; // Hidden from user -}>; - -// Type guard -export const isFeatureSchemaComponent = ( -comp: AnySchemaComponent -): comp is FeatureSchemaComponent => { -return '__featureMarker' in comp && comp.__featureMarker === true; -}; -Event Store Schema Component -export type EventStoreSchemaComponent = FeatureSchemaComponent< -'sc:dumbo:feature:event_store', -{ - eventStoreName?: string; - inlineProjections?: Record; -} ->; - -export const eventStoreSchema = ( -options?: { - inlineProjections?: Record; -} -): EventStoreSchemaComponent => { -// Create internal tables -const streams = dumboTable('streams', { - migrations: [sqlMigration('create_streams', [streamsTableSQL])] -}); -const messages = dumboTable('messages', { - migrations: [sqlMigration('create_messages', [messagesTableSQL])] -}); -const subscriptions = dumboTable('subscriptions', { - migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])] -}); - -// Handle inline projections -const projectionTables = Object.entries(options?.inlineProjections ?? {}).map( - ([name, collectionSchema]) => dumboTable(collectionSchema.name, {}) -); - -const allComponents = [streams, messages, subscriptions, ...projectionTables]; - -const base = schemaComponent('sc:dumbo:feature:event_store', { - components: allComponents, -}); - -return { - ...base, - __featureMarker: true as const, - eventStoreName: 'event_store', - inlineProjections: options?.inlineProjections, - internalComponents: new Map(allComponents.map(c => [c.schemaComponentKey, c])), -}; -}; -Pongo Collections Schema Component -export type PongoCollectionsSchemaComponent = FeatureSchemaComponent< -'sc:dumbo:feature:pongo_collections', -{ - pongoSchema: PongoDbSchema; -} ->; - -export const pongoCollectionsSchema = ( -collections: Record -): PongoCollectionsSchemaComponent => { -// Create dumbo table for each pongo collection -const tables = Object.entries(collections).map(([_name, collectionSchema]) => - dumboTable(collectionSchema.name, { - // Auto-generate basic table structure for Pongo - // Could add _id, document JSONB columns, etc. - }) -); - -const base = schemaComponent('sc:dumbo:feature:pongo_collections', { - components: tables, -}); - -return { - ...base, - __featureMarker: true as const, - pongoSchema: pongoSchema.db(collections), - internalComponents: new Map(tables.map(t => [t.schemaComponentKey, t])), -}; -}; -Migration Collection -// In migration orchestrator -export const collectAllMigrations = ( -database: DatabaseSchemaComponent, -): SQLMigration[] => { -const migrations: SQLMigration[] = []; - -for (const schema of database.schemas.values()) { - for (const component of schema.components.values()) { - if (isFeatureSchemaComponent(component)) { - // Feature component: extract internal components - for (const internalComp of component.internalComponents.values()) { - if (isTableComponent(internalComp)) { - migrations.push(...internalComp.migrations); - } - } - } else if (isTableComponent(component)) { - // Regular table - migrations.push(...component.migrations); - } - } -} - -return migrations; -}; -Final Clarifying Questions -Q1: Component Placement -Which feels most natural? -// A: Mixed (type system differentiates) -dumboDatabaseSchema('public', { -eventStore: eventStoreSchema(), -guests: dumboTable('guests', {}), -}) - -// B: Separated -dumboDatabaseSchema('public', { -features: { eventStore: eventStoreSchema() }, -tables: { guests: dumboTable('guests', {}) }, -}) - -// C: Builder pattern -dumboDatabaseSchema('public') -.withFeature(eventStoreSchema()) -.withTable(dumboTable('guests', {})) -Q2: Generated Types Visibility -Should generated types expose event store internals? -// Option A: Expose everything -type Schema = { -public: { - streams: { ... }, // From event store - messages: { ... }, // From event store - guests: { ... }, // From user -} -} - -// Option B: Keep features opaque -type Schema = { -public: { - guests: { ... }, // Only user-defined tables -} -} -Q3: Pongo Collections Table Structure -When pongoCollectionsSchema() creates tables, what should the table structure be? -// Option A: Document store (MongoDB-like) -CREATE TABLE guests ( -_id TEXT PRIMARY KEY, -document JSONB NOT NULL -) - -// Option B: Let user define structure -pongoCollectionsSchema({ -guests: pongoSchema.collection('guests', { - tableDefinition: dumboTable('guests', { - columns: { id, name, email, ... } - }) -}) -}) - -// Option C: Infer from Pongo usage later -Please confirm these choices and I'll build the complete solution! -$$ diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts index d67b99eb..9aec74f4 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.ts @@ -1,5 +1,5 @@ import type { AnyColumnTypeToken, SQLColumnToken } from '../../sql'; -import type { ValidateDatabaseRelationships } from '../components'; +import type { ValidateDatabaseSchemas } from '../components'; import { type AnyDatabaseSchemaSchemaComponent, columnSchemaComponent, @@ -28,24 +28,29 @@ const DEFAULT_DATABASE_NAME = '__default_database__'; const DEFAULT_DATABASE_SCHEMA_NAME = '__default_database_schema__'; const dumboColumn = < - ColumnType extends AnyColumnTypeToken | string = AnyColumnTypeToken | string, - TOptions extends SchemaComponentOptions & + const ColumnType extends AnyColumnTypeToken | string = + | AnyColumnTypeToken + | string, + const TOptions extends SchemaComponentOptions & Omit, 'name' | 'type' | 'sqlTokenType'> = Omit< ColumnSchemaComponentOptions, 'type' >, + const ColumnName extends string = string, >( - name: string, + name: ColumnName, type: ColumnType, options?: TOptions, -): ReturnType< - typeof columnSchemaComponent -> => - columnSchemaComponent({ +) => + columnSchemaComponent< + ColumnType, + TOptions & { type: ColumnType }, + ColumnName + >({ columnName: name, type, ...options, - } as { columnName: string } & TOptions & { type: ColumnType }); + } as { columnName: ColumnName } & TOptions & { type: ColumnType }); const dumboIndex = ( name: string, @@ -60,21 +65,22 @@ const dumboIndex = ( }); const dumboTable = < - Columns extends TableColumns = TableColumns, + const Columns extends TableColumns = TableColumns, + const TableName extends string = string, const Relationships extends TableRelationships< keyof Columns & string > = TableRelationships, >( - name: string, + name: TableName, definition: { columns?: Columns; primaryKey?: TableColumnNames< - TableSchemaComponent + TableSchemaComponent >[]; relationships?: Relationships; indexes?: Record; } & SchemaComponentOptions, -): TableSchemaComponent => { +): TableSchemaComponent => { const { columns, indexes, primaryKey, relationships, ...options } = definition; @@ -91,26 +97,30 @@ const dumboTable = < }; function dumboDatabaseSchema< - Tables extends DatabaseSchemaTables = DatabaseSchemaTables, ->(tables: Tables): DatabaseSchemaSchemaComponent; + const Tables extends DatabaseSchemaTables = DatabaseSchemaTables, +>( + tables: Tables, +): DatabaseSchemaSchemaComponent; function dumboDatabaseSchema< - Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const SchemaName extends string = string, >( - schemaName: string, + schemaName: SchemaName, tables: Tables, options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent; +): DatabaseSchemaSchemaComponent; function dumboDatabaseSchema< - Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const Tables extends DatabaseSchemaTables = DatabaseSchemaTables, + const SchemaName extends string = string, >( - nameOrTables: string | Tables, + nameOrTables: SchemaName | Tables, tables?: Tables, options?: SchemaComponentOptions, -): DatabaseSchemaSchemaComponent { +): DatabaseSchemaSchemaComponent { const schemaName = typeof nameOrTables === 'string' ? nameOrTables - : DEFAULT_DATABASE_SCHEMA_NAME; + : (DEFAULT_DATABASE_SCHEMA_NAME as SchemaName); const tablesMap = (typeof nameOrTables === 'string' ? tables : nameOrTables) ?? ({} as Tables); @@ -141,11 +151,16 @@ dumboDatabaseSchema.from = ( type ValidatedDatabaseSchemaComponent< Schemas extends DatabaseSchemas = DatabaseSchemas, > = - ValidateDatabaseRelationships> extends { + ValidateDatabaseSchemas extends { valid: true; } ? DatabaseSchemaComponent - : ValidateDatabaseRelationships>; + : ValidateDatabaseSchemas extends { + valid: false; + error: infer E; + } + ? { valid: false; error: E } + : DatabaseSchemaComponent; function dumboDatabase( schemas: Schemas, @@ -199,7 +214,7 @@ function dumboDatabase( databaseName, schemas: schemaMap as Schemas, ...dbOptions, - }); + }) as ValidatedDatabaseSchemaComponent; } dumboDatabase.from = ( diff --git a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts index 6baff728..633aa469 100644 --- a/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts +++ b/src/packages/dumbo/src/core/schema/dumboSchema/dumboSchema.unit.spec.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { SQL } from '../../sql'; -import type { Equal, Expect } from '../../testing'; +import type { Equals, Expect } from '../../testing'; import type { TableColumnNames, TableRowType } from '../components'; import { relationship } from '../components'; import { dumboSchema } from './index'; @@ -215,11 +215,11 @@ const _usersTable = publicSchema.tables.users; type Users = TableRowType; -type _IdColumnIsNonNullableString = Expect>; -type _EmailColumnIsNonNullableString = Expect>; -type _NameColumnIsNullableString = Expect>; +type _IdColumnIsNonNullableString = Expect>; +type _EmailColumnIsNonNullableString = Expect>; +type _NameColumnIsNullableString = Expect>; type _MetadataColumnIsNullableObject = Expect< - Equal + Equals >; type UserColumns = TableColumnNames; diff --git a/src/packages/dumbo/src/core/schema/schemaComponent.ts b/src/packages/dumbo/src/core/schema/schemaComponent.ts index 78d4b713..3d199291 100644 --- a/src/packages/dumbo/src/core/schema/schemaComponent.ts +++ b/src/packages/dumbo/src/core/schema/schemaComponent.ts @@ -63,7 +63,7 @@ export type SchemaComponentType = `sc:${Kind}`; export type DumboSchemaComponentType = SchemaComponentType<`dumbo:${Kind}`>; -export const schemaComponent = ( +export const schemaComponent = ( key: ComponentKey, options: SchemaComponentOptions, ): SchemaComponent => { diff --git a/src/packages/dumbo/src/core/schema/schema_feature.md b/src/packages/dumbo/src/core/schema/schema_feature.md deleted file mode 100644 index 4edf5f06..00000000 --- a/src/packages/dumbo/src/core/schema/schema_feature.md +++ /dev/null @@ -1,1345 +0,0 @@ -# Schema Feature Component Design - -## Overview - -This document defines the design for **Feature Schema Components** - a composable, opaque abstraction layer for complex database features like event stores, Pongo collections, and custom application features that can be integrated seamlessly into Dumbo's schema system. - -## Problem Statement - -Current Dumbo schema components (`database`, `schema`, `table`, `column`, `index`) work well for explicit table definitions, but pose challenges for: - -1. **Complex Features**: Features like event stores have multiple tables, functions, and internal structure that users shouldn't need to understand -2. **Framework Integration**: Pongo collections need both Dumbo tables (for migrations) and Pongo schema (for querying) -3. **Multi-Database Support**: Same schema definition should work across PostgreSQL (with schemas), SQLite (without schemas), and other databases -4. **Multi-Tenancy**: Need flexible sharding strategies applicable at different levels (table, schema, database) -5. **System Metadata**: Migration tracking and schema introspection tables need consistent, configurable placement - -## Design Principles - -1. **Opaque by Default**: Users don't need to know internal structure of features (e.g., event store tables) -2. **Composable**: Features compose like other schema components -3. **Strongly Typed**: Generated types expose internal structure for type-safe queries -4. **Database Agnostic**: Schema definitions are portable; migration generation is database-specific -5. **Extensible**: Generic sharding mechanism supports multi-tenancy and custom partitioning strategies -6. **Explicit When Needed**: Power users can customize feature internals and placement - -## Core Concepts - -### 1. Feature Schema Component - -A feature is an opaque schema component that encapsulates internal components (tables, functions, etc.). - -```typescript -// Base marker interface for all features -export type FeatureSchemaComponent< - URN extends string = string, - AdditionalData extends Record = Record, -> = SchemaComponent< - URN, - AdditionalData & { - __featureMarker: true; - internalComponents: ReadonlyMap; // Hidden from user API - } ->; - -// Type guard -export const isFeatureSchemaComponent = ( - comp: AnySchemaComponent, -): comp is FeatureSchemaComponent => { - return '__featureMarker' in comp && comp.__featureMarker === true; -}; -``` - -### 2. Event Store Schema Component - -Event store as a first-class feature component. - -```typescript -export type EventStoreSchemaComponent = FeatureSchemaComponent< - 'sc:dumbo:feature:event_store', - { - eventStoreName?: string; - inlineProjections?: Record; - } ->; - -export const eventStoreSchema = (options?: { - inlineProjections?: Record; -}): EventStoreSchemaComponent => { - // Create internal tables (opaque to user) - const streams = dumboTable('streams', { - migrations: [sqlMigration('create_streams', [streamsTableSQL])], - }); - - const messages = dumboTable('messages', { - migrations: [sqlMigration('create_messages', [messagesTableSQL])], - }); - - const subscriptions = dumboTable('subscriptions', { - migrations: [sqlMigration('create_subscriptions', [subscriptionsTableSQL])], - }); - - // Create functions - const appendFunction = dumboFunction('emt_append_to_stream', { - migrations: [sqlMigration('create_append_function', [appendToStreamSQL])], - }); - - // Handle inline projections (create tables from Pongo collections) - const projectionTables = Object.entries(options?.inlineProjections ?? {}).map( - ([name, collectionSchema]) => - dumboTable(collectionSchema.name, { - // Auto-generate table structure for Pongo collection - migrations: [ - sqlMigration(`create_projection_${name}`, [ - // Generate CREATE TABLE with _id and document columns - ]), - ], - }), - ); - - const allComponents = [ - streams, - messages, - subscriptions, - appendFunction, - ...projectionTables, - ]; - - const base = schemaComponent('sc:dumbo:feature:event_store', { - components: allComponents, - }); - - return { - ...base, - __featureMarker: true as const, - eventStoreName: 'event_store', - inlineProjections: options?.inlineProjections, - internalComponents: new Map( - allComponents.map((c) => [c.schemaComponentKey, c]), - ), - }; -}; -``` - -### 3. Pongo Collections Schema Component - -Bridges Pongo's collection abstraction with Dumbo's table-based migrations. - -```typescript -export type PongoCollectionsSchemaComponent = FeatureSchemaComponent< - 'sc:dumbo:feature:pongo_collections', - { - pongoSchema: PongoDbSchema; - } ->; - -export const pongoCollectionsSchema = ( - collections: Record, -): PongoCollectionsSchemaComponent => { - // Create Dumbo table for each Pongo collection - const tables = Object.entries(collections).map(([_name, collectionSchema]) => - dumboTable(collectionSchema.name, { - migrations: [ - sqlMigration(`create_collection_${collectionSchema.name}`, [ - // Auto-generate table structure: - // CREATE TABLE {name} ( - // _id TEXT PRIMARY KEY, - // document JSONB NOT NULL, - // created_at TIMESTAMP DEFAULT NOW(), - // updated_at TIMESTAMP DEFAULT NOW() - // ) - ]), - ], - }), - ); - - const base = schemaComponent('sc:dumbo:feature:pongo_collections', { - components: tables, - }); - - return { - ...base, - __featureMarker: true as const, - pongoSchema: pongoSchema.db(collections), - internalComponents: new Map(tables.map((t) => [t.schemaComponentKey, t])), - }; -}; -``` - -### 4. System Schema Component - -Contains framework metadata tables (migrations, pongo metadata, schema introspection). - -```typescript -export type SystemSchemaComponent = FeatureSchemaComponent< - 'sc:dumbo:feature:system', - { - systemTables: { - migrationTracking: boolean; - pongoMetadata: boolean; - schemaIntrospection: boolean; - }; - } ->; - -export function systemSchema(): SystemSchemaComponent; -export function systemSchema(schemaName: string): DatabaseSchemaSchemaComponent; -export function systemSchema(options: { - migrationTracking?: boolean; - pongoMetadata?: boolean; - schemaIntrospection?: boolean; -}): SystemSchemaComponent; -export function systemSchema( - nameOrOptions?: - | string - | { - migrationTracking?: boolean; - pongoMetadata?: boolean; - schemaIntrospection?: boolean; - }, -): SystemSchemaComponent | DatabaseSchemaSchemaComponent { - const options = - typeof nameOrOptions === 'string' - ? { schemaName: nameOrOptions } - : nameOrOptions; - - const tables: TableSchemaComponent[] = []; - - if (options?.migrationTracking !== false) { - tables.push( - dumboTable('__migrations', { - migrations: [ - sqlMigration('create_migrations_table', [ - SQL`CREATE TABLE IF NOT EXISTS __migrations ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - applied_at TIMESTAMP DEFAULT NOW() - )`, - ]), - ], - }), - ); - } - - if (options?.pongoMetadata !== false) { - tables.push( - dumboTable('__pongo_collections', { - migrations: [ - sqlMigration('create_pongo_metadata_table', [ - SQL`CREATE TABLE IF NOT EXISTS __pongo_collections ( - collection_name TEXT PRIMARY KEY, - json_schema JSONB, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() - )`, - ]), - ], - }), - ); - } - - if (options?.schemaIntrospection !== false) { - tables.push( - dumboTable('__schema_metadata', { - migrations: [ - sqlMigration('create_schema_metadata_table', [ - SQL`CREATE TABLE IF NOT EXISTS __schema_metadata ( - component_type TEXT NOT NULL, - component_name TEXT NOT NULL, - component_key TEXT PRIMARY KEY, - definition JSONB NOT NULL, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() - )`, - ]), - ], - }), - ); - } - - const base = schemaComponent('sc:dumbo:feature:system', { - components: tables, - }); - - const component = { - ...base, - __featureMarker: true as const, - systemTables: { - migrationTracking: options?.migrationTracking !== false, - pongoMetadata: options?.pongoMetadata !== false, - schemaIntrospection: options?.schemaIntrospection !== false, - }, - internalComponents: new Map(tables.map((t) => [t.schemaComponentKey, t])), - } satisfies SystemSchemaComponent; - - // If schemaName provided, wrap in schema component - if ('schemaName' in options && options.schemaName) { - return dumboDatabaseSchema(options.schemaName, {}).addComponent(component); - } - - return component; -} - -export const isSystemSchemaComponent = ( - comp: AnySchemaComponent, -): comp is SystemSchemaComponent => { - return comp.schemaComponentKey === 'sc:dumbo:feature:system'; -}; -``` - -### 5. Function Schema Component - -First-class support for stored procedures/functions. - -```typescript -export type FunctionURNType = 'sc:dumbo:function'; -export type FunctionURN = `${FunctionURNType}:${string}`; - -export type FunctionSchemaComponent = SchemaComponent< - FunctionURN, - Readonly<{ - functionName: string; - language?: 'plpgsql' | 'sql' | 'javascript'; // Database-specific - }> ->; - -export const dumboFunction = ( - name: string, - options: SchemaComponentOptions & { - language?: 'plpgsql' | 'sql' | 'javascript'; - }, -): FunctionSchemaComponent => { - const base = schemaComponent(`sc:dumbo:function:${name}` as FunctionURN, { - migrations: options.migrations ?? [], - components: options.components ?? [], - }); - - return { - ...base, - functionName: name, - language: options.language, - }; -}; -``` - -## Usage Examples - -### Example 1: Simple Single-Schema Application - -```typescript -// Everything in one schema, system tables auto-added -const simpleApp = dumboDatabase('my_app', { - public: dumboDatabaseSchema('public', { - users: dumboTable('users', { - columns: { - id: dumboColumn('id', { type: 'serial', primaryKey: true }), - email: dumboColumn('email', { type: 'varchar', length: 255 }), - }, - }), - posts: dumboTable('posts', { - columns: { - id: dumboColumn('id', { type: 'serial', primaryKey: true }), - userId: dumboColumn('user_id', { type: 'int' }), - }, - }), - }), -}); - -// System tables (__migrations, __pongo_collections, __schema_metadata) -// are automatically added to the 'public' schema -``` - -### Example 2: Event Store with Read Models - -```typescript -// Event store and read models in same schema -const hotelApp = dumboDatabase('hotel', { - public: dumboDatabaseSchema('public', { - // Event store feature (opaque - user doesn't see internal tables) - eventStore: eventStoreSchema(), - - // User-defined tables - guests: dumboTable('guests', { - columns: { - id: dumboColumn('id', { type: 'serial', primaryKey: true }), - name: dumboColumn('name', { type: 'varchar', length: 255 }), - }, - }), - - reservations: dumboTable('reservations', { - columns: { - id: dumboColumn('id', { type: 'serial', primaryKey: true }), - guestId: dumboColumn('guest_id', { type: 'int' }), - }, - }), - }), -}); - -// Generated migrations include: -// - System tables: __migrations, __pongo_collections, __schema_metadata -// - Event store tables: streams, messages, subscriptions -// - Event store function: emt_append_to_stream -// - User tables: guests, reservations -``` - -### Example 3: Separated Schemas - -```typescript -// Event store and read models in separate schemas -const hotelApp = dumboDatabase('hotel', { - // Dedicated system schema - system: systemSchema('system'), - - // Event store in its own schema - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - - // Read models in separate schema - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { - /* ... */ - }), - reservations: dumboTable('reservations', { - /* ... */ - }), - }), -}); - -// PostgreSQL generates: -// CREATE SCHEMA system; -// CREATE TABLE system.__migrations (...); -// CREATE SCHEMA event_store; -// CREATE TABLE event_store.streams (...); -// CREATE SCHEMA read_models; -// CREATE TABLE read_models.guests (...); -``` - -### Example 4: Event Store with Inline Projections - -```typescript -const hotelApp = dumboDatabase('hotel', { - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema({ - // Inline projections become tables in same schema - inlineProjections: { - guestSummary: pongoSchema.collection('guest_summary'), - reservationSummary: pongoSchema.collection('reservation_summary'), - }, - }), - }), - - read_models: dumboDatabaseSchema('read_models', { - // Or define Pongo collections as a feature - collections: pongoCollectionsSchema({ - guests: pongoSchema.collection('guests'), - reservations: pongoSchema.collection('reservations'), - }), - }), -}); - -// event_store schema contains: -// - streams, messages, subscriptions (event store core) -// - guest_summary, reservation_summary (inline projections) -// -// read_models schema contains: -// - guests, reservations (Pongo collections → Dumbo tables) -``` - -### Example 5: Multiple Databases with Database Group - -```typescript -// Database group for organizing related databases -const hotelSystemGroup = databaseGroup( - 'hotel_system', - { - // Operational database - operational: dumboDatabase('hotel_operational', { - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { - /* ... */ - }), - reservations: dumboTable('reservations', { - /* ... */ - }), - }), - }), - - // Analytics database - analytics: dumboDatabase('hotel_analytics', { - public: dumboDatabaseSchema('public', { - events: dumboTable('events', { - /* ... */ - }), - metrics: dumboTable('metrics', { - /* ... */ - }), - }), - }), - - // Reporting database - reporting: dumboDatabase('hotel_reporting', { - public: dumboDatabaseSchema('public', { - reports: dumboTable('reports', { - /* ... */ - }), - }), - }), - }, - { - // Shared system schema across all databases - shared: { - systemSchema: systemSchema('shared_system'), - }, - }, -); -``` - -## System Schema Placement Strategy - -### Single Schema - -When database has exactly one schema, system tables are automatically added to that schema. - -```typescript -const db = dumboDatabase('app', { - public: dumboDatabaseSchema('public', { - /* ... */ - }), -}); -// ✅ System tables → 'public' schema -``` - -### Multiple Schemas Without Explicit System Schema - -Default behavior: Use database type's default schema convention. - -```typescript -const db = dumboDatabase('app', { - event_store: dumboDatabaseSchema('event_store', { - /* ... */ - }), - read_models: dumboDatabaseSchema('read_models', { - /* ... */ - }), -}); - -// PostgreSQL: Check for 'public' schema, otherwise use first schema -// ✅ If 'public' exists → system tables go there -// ✅ Otherwise → system tables go to 'event_store' (first schema) - -// SQLite: Use first schema (schema names become table prefixes) -// ✅ System tables → 'event_store' schema (prefixed: event_store___migrations) -``` - -### Explicit System Schema - -User can explicitly place system schema anywhere: - -```typescript -// Option A: Dedicated system schema -const db = dumboDatabase('app', { - system: systemSchema('admin'), // Returns DatabaseSchemaSchemaComponent - event_store: dumboDatabaseSchema('event_store', { - /* ... */ - }), - read_models: dumboDatabaseSchema('read_models', { - /* ... */ - }), -}); - -// Option B: System schema within existing schema -const db = dumboDatabase('app', { - event_store: dumboDatabaseSchema('event_store', { - system: systemSchema(), // Just the component - eventStore: eventStoreSchema(), - }), - read_models: dumboDatabaseSchema('read_models', { - /* ... */ - }), -}); -``` - -## Generic Sharding Strategy - -### Sharding Configuration - -```typescript -export type ShardingStrategy = { - // How to identify this shard dimension - shardKey: TShardKey; - - // What values this shard can have - shardValues: readonly string[] | 'dynamic'; - - // Naming pattern for the sharded resource - namingPattern: string | ((shard: string, original: string) => string); - - // Which resources should NOT be sharded (shared across all shards) - exclude?: string[] | ((resourceName: string) => boolean); -}; - -export type ShardingLevel = 'table' | 'schema' | 'database' | 'database-group'; - -export type ShardingConfig = { - level: ShardingLevel; - strategy: ShardingStrategy; - - // Optional: Apply sharding only to specific components - applyTo?: string[]; -}; -``` - -### Predefined Sharding Strategies - -```typescript -export const shardingStrategies = { - // Multi-tenancy: Shard by tenant ID - multiTenant: (tenants: string[] | 'dynamic'): ShardingStrategy => ({ - shardKey: 'tenant_id', - shardValues: tenants, - namingPattern: '{shard}_{resource}', - exclude: ['system'], // Don't shard system schema - }), - - // Geographic regions - region: (regions: string[]): ShardingStrategy => ({ - shardKey: 'region', - shardValues: regions, - namingPattern: '{resource}_{shard}', - }), - - // Time-based partitioning - timePartition: (periods: string[]): ShardingStrategy => ({ - shardKey: 'period', - shardValues: periods, // e.g., ['2024_01', '2024_02', ...] - namingPattern: '{resource}_{shard}', - }), - - // Customer-based sharding - customerId: (customerIds: string[] | 'dynamic'): ShardingStrategy => ({ - shardKey: 'customer_id', - shardValues: customerIds, - namingPattern: 'cust_{shard}_{resource}', - }), -}; -``` - -### Sharding Examples - -#### Schema-Level Sharding (Multi-Tenancy) - -```typescript -const hotelApp = dumboDatabase( - 'hotel', - { - system: systemSchema('system'), // Not sharded - - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { - /* ... */ - }), - reservations: dumboTable('reservations', { - /* ... */ - }), - }), - }, - { - sharding: { - level: 'schema', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott', 'hyatt']), - }, - }, -); - -// PostgreSQL generates: -// CREATE SCHEMA system; -- Not sharded -// CREATE SCHEMA hilton_event_store; -// CREATE TABLE hilton_event_store.streams (...); -// CREATE SCHEMA hilton_read_models; -// CREATE TABLE hilton_read_models.guests (...); -// CREATE SCHEMA marriott_event_store; -// CREATE TABLE marriott_event_store.streams (...); -// CREATE SCHEMA marriott_read_models; -// CREATE TABLE marriott_read_models.guests (...); -// ... -``` - -#### Database-Level Sharding - -```typescript -const hotelApp = dumboDatabase( - 'hotel', - { - event_store: dumboDatabaseSchema('event_store', { - /* ... */ - }), - read_models: dumboDatabaseSchema('read_models', { - /* ... */ - }), - }, - { - sharding: { - level: 'database', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), - }, - }, -); - -// PostgreSQL generates separate databases: -// CREATE DATABASE hilton_hotel; -// CREATE DATABASE marriott_hotel; - -// SQLite generates separate files: -// hilton_hotel.db -// marriott_hotel.db -``` - -#### Table-Level Sharding (Time Partitioning) - -```typescript -const analyticsDb = dumboDatabase( - 'analytics', - { - public: dumboDatabaseSchema('public', { - events: dumboTable('events', { - /* ... */ - }), - metrics: dumboTable('metrics', { - /* ... */ - }), - }), - }, - { - sharding: { - level: 'table', - strategy: shardingStrategies.timePartition([ - '2024_01', - '2024_02', - '2024_03', - ]), - applyTo: ['events'], // Only shard events table - }, - }, -); - -// Generates: -// CREATE TABLE events_2024_01 (...); -// CREATE TABLE events_2024_02 (...); -// CREATE TABLE events_2024_03 (...); -// CREATE TABLE metrics (...); -- Not sharded -``` - -#### Database Group Sharding - -```typescript -const hotelSystemGroup = databaseGroup( - 'hotel_system', - { - operational: dumboDatabase('hotel_operational', { - /* ... */ - }), - analytics: dumboDatabase('hotel_analytics', { - /* ... */ - }), - reporting: dumboDatabase('hotel_reporting', { - /* ... */ - }), - }, - { - sharding: { - level: 'database', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), - }, - shared: { - systemSchema: systemSchema('shared_system'), - }, - }, -); - -// Generates for each tenant: -// Tenant 'hilton': -// Database: hilton_hotel_operational -// Database: hilton_hotel_analytics -// Database: hilton_hotel_reporting -// Shared: shared_system schema (referenced from all databases) -// -// Tenant 'marriott': -// Database: marriott_hotel_operational -// Database: marriott_hotel_analytics -// Database: marriott_hotel_reporting -// Shared: shared_system schema (referenced from all databases) -``` - -### Component-Level Sharding Override - -```typescript -// Different sharding strategies for different schemas -const hotelApp = dumboDatabase('hotel', { - system: systemSchema('system'), // No sharding - - event_store: dumboDatabaseSchema( - 'event_store', - { - eventStore: eventStoreSchema(), - }, - { - // Override: shard event store by tenant - sharding: { - level: 'schema', - strategy: shardingStrategies.multiTenant(['hilton', 'marriott']), - }, - }, - ), - - analytics: dumboDatabaseSchema( - 'analytics', - { - events: dumboTable('events', { - /* ... */ - }), - }, - { - // Override: shard analytics by region - sharding: { - level: 'table', - strategy: shardingStrategies.region(['us_east', 'us_west', 'eu']), - }, - }, - ), -}); - -// Generates: -// CREATE SCHEMA system; -// CREATE SCHEMA hilton_event_store; -// CREATE SCHEMA marriott_event_store; -// CREATE SCHEMA analytics; -// CREATE TABLE analytics.events_us_east (...); -// CREATE TABLE analytics.events_us_west (...); -// CREATE TABLE analytics.events_eu (...); -``` - -## Database Group Design - -### Database Group Component - -```typescript -export type DatabaseGroup< - Databases extends Record = Record< - string, - DatabaseSchemaComponent - >, -> = { - groupName: string; - databases: Databases; - - // Group-level sharding configuration - sharding?: ShardingConfig; - - // Resources shared across all databases in group - shared?: { - systemSchema?: SystemSchemaComponent; - }; -}; - -export const databaseGroup = < - T extends Record, ->( - groupName: string, - databases: T, - options?: { - sharding?: ShardingConfig; - shared?: { - systemSchema?: SystemSchemaComponent; - }; - }, -): DatabaseGroup => ({ - groupName, - databases, - sharding: options?.sharding, - shared: options?.shared, -}); -``` - -### Use Cases for Database Groups - -1. **Logical Organization**: Group related databases for documentation and architecture diagrams -2. **Shared Sharding**: Apply same sharding strategy across multiple databases -3. **Shared Resources**: Single system schema referenced by multiple databases -4. **CQRS/Event Sourcing**: Separate databases for commands, queries, and events -5. **Polyglot Persistence**: Different databases for different concerns (operational, analytics, reporting) - -## Migration Generation - -### Database-Agnostic Schema, Database-Specific Migrations - -Schema definitions are portable; migration generation considers database type: - -```typescript -// Schema definition (database-agnostic) -const hotelSchema = dumboDatabase('hotel', { - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { - /* ... */ - }), - }), -}); - -// PostgreSQL migration generation -const pgMigrations = generateMigrations(hotelSchema, { - databaseType: 'postgresql', -}); -// Generates: -// CREATE SCHEMA event_store; -// CREATE TABLE event_store.streams (...); -// CREATE SCHEMA read_models; -// CREATE TABLE read_models.guests (...); - -// SQLite migration generation -const sqliteMigrations = generateMigrations(hotelSchema, { - databaseType: 'sqlite', - sqliteStrategy: 'prefix-tables', // or 'separate-files' -}); -// prefix-tables generates: -// CREATE TABLE event_store_streams (...); -// CREATE TABLE read_models_guests (...); -// -// separate-files generates: -// File: event_store.db → CREATE TABLE streams (...); -// File: read_models.db → CREATE TABLE guests (...); -``` - -### Migration Collection from Features - -The migration orchestrator detects feature components and extracts their internal migrations: - -```typescript -export const collectAllMigrations = ( - database: DatabaseSchemaComponent, -): SQLMigration[] => { - const migrations: SQLMigration[] = []; - - for (const schema of database.schemas.values()) { - for (const component of schema.components.values()) { - if (isFeatureSchemaComponent(component)) { - // Feature component: extract internal components - for (const internalComp of component.internalComponents.values()) { - if (isTableComponent(internalComp)) { - migrations.push(...internalComp.migrations); - } else if (isFunctionComponent(internalComp)) { - migrations.push(...internalComp.migrations); - } - } - } else if (isTableComponent(component)) { - // Regular table - migrations.push(...component.migrations); - } else if (isFunctionComponent(component)) { - // Regular function - migrations.push(...component.migrations); - } - } - } - - return migrations; -}; -``` - -## Type Generation - -### Exposing Feature Internals in Generated Types - -While features are opaque at definition time, generated types expose internal structure for type-safe queries: - -```typescript -// Schema definition (event store is opaque) -const hotelSchema = dumboDatabase('hotel', { - public: dumboDatabaseSchema('public', { - eventStore: eventStoreSchema(), - guests: dumboTable('guests', { - /* ... */ - }), - }), -}); - -// Generated types expose all tables (including event store internals) -export type HotelSchema = { - public: { - // Event store tables (exposed for type-safe queries) - streams: { - stream_id: string; - stream_position: number; - partition: string; - stream_type: string; - stream_metadata: unknown; - is_archived: boolean; - }; - messages: { - stream_id: string; - stream_position: number; - partition: string; - message_kind: string; - message_data: unknown; - message_metadata: unknown; - message_schema_version: string; - message_type: string; - message_id: string; - is_archived: boolean; - global_position: number; - created: Date; - }; - subscriptions: { - subscription_id: string; - version: number; - partition: string; - last_processed_position: number; - }; - - // User-defined tables - guests: { - id: number; - name: string; - email: string; - }; - }; -}; - -// Usage with type-safe queries -const stream = await db - .from('streams') // TypeScript knows this exists - .where('stream_id', '=', 'guest-123') - .select(['stream_position', 'stream_type']); -// Type: { stream_position: number, stream_type: string }[] -``` - -## Integration with Emmett Architecture - -Feature components are separate from Emmett architectural components but can be mapped: - -```typescript -// Emmett: Logical architecture (business components with ports) -const hotelManagementContainer = emmettArch.container('hotel-management', { - eventStore: emmettArch.component('event-store', { - ports: { - exposes: { - commands: { - appendToStream: (/* ... */) => Promise.resolve(), - }, - }, - }, - }), - - guests: emmettArch.component('guests', { - ports: { - exposes: { - queries: { - getGuestByExternalId: query(), - }, - }, - }, - }), - - reservations: emmettArch.component('reservations', { - ports: { - requires: { - guests: { - getGuestByExternalId: query(), - }, - }, - }, - }), -}); - -// Dumbo: Physical schema (database structure) -const hotelSchema = dumboDatabase('hotel', { - event_store: dumboDatabaseSchema('event_store', { - eventStore: eventStoreSchema(), - }), - - read_models: dumboDatabaseSchema('read_models', { - guests: dumboTable('guests', { - /* ... */ - }), - reservations: dumboTable('reservations', { - /* ... */ - }), - }), -}); - -// Mapping layer (optional - for documentation/tooling) -const deployment = { - architecture: hotelManagementContainer, - schema: hotelSchema, - mapping: { - // Map Emmett components to Dumbo schemas/tables - 'event-store': 'event_store.eventStore', - guests: 'read_models.guests', - reservations: 'read_models.reservations', - }, -}; -``` - -## Design Decisions - -### 1. Why Feature Components Extend SchemaComponent? - -- **Uniform Composition**: Features compose like tables, indexes, etc. -- **Consistent API**: Same `addComponent()` pattern everywhere -- **Type Safety**: Generic `SchemaComponent` infrastructure works for features -- **Migration System**: Features participate in migration collection automatically - -### 2. Why System Schema is a Feature Component? - -- **Consistent Placement**: Same composition rules as other features -- **Flexible Location**: Can be in dedicated schema or mixed with application schemas -- **Opaque Internals**: Users don't need to know about **migrations, **pongo_collections tables -- **Customizable**: Power users can configure which system tables to include - -### 3. Why Generic Sharding vs. Hardcoded Multi-Tenancy? - -- **Extensibility**: Supports time partitioning, regional sharding, custom strategies -- **Composition**: Can combine multiple sharding dimensions in the future -- **Flexibility**: Same mechanism for different use cases (multi-tenancy, scaling, compliance) -- **Simplicity**: Single concept to learn instead of multiple special cases - -### 4. Why Database Groups? - -- **Organization**: Large systems have multiple related databases -- **Shared Resources**: System schema can be shared across databases -- **Consistent Sharding**: Apply same tenant strategy to command/query/event databases -- **Documentation**: Architecture diagrams show logical database groupings - -### 5. Why Portable Schema Definitions? - -- **Developer Experience**: Write once, deploy to PostgreSQL or SQLite -- **Testing**: Test with SQLite, deploy to PostgreSQL -- **Flexibility**: Change database type without rewriting schema -- **Separation of Concerns**: Schema = logical structure, migrations = physical implementation - -## Future Enhancements - -### 1. Composable Sharding (Out of Scope for Initial Implementation) - -Support multiple sharding dimensions: - -```typescript -{ - sharding: [ - { level: 'database', strategy: shardingStrategies.multiTenant(['hilton']) }, - { level: 'schema', strategy: shardingStrategies.region(['us', 'eu']) }, - { level: 'table', strategy: shardingStrategies.timePartition(['2024_01']) }, - ]; -} -// Generates: Database hilton_hotel → Schema us_event_store → Table events_2024_01 -``` - -### 2. Pattern-Based Exclusion (Future) - -Support patterns in sharding exclusion: - -```typescript -{ - sharding: { - strategy: tenantSharding, - exclude: [ - 'system', // Exact match - '__*', // Glob pattern: all system tables - /^temp_/, // Regex: temporary tables - (name) => name.startsWith('cache_'), // Function - ] - } -} -``` - -### 3. Dynamic Shard Provisioning (Out of Scope for Schema Definition) - -Runtime provisioning of new shards (tenants, regions, etc.): - -```typescript -// This is a runtime concern, not schema definition -const provisioner = createShardProvisioner(schema, config); -await provisioner.provisionShard(connection, 'new_tenant_id'); -``` - -### 4. Cross-Database Relationships (Future) - -Support foreign keys across sharded databases: - -```typescript -// Define relationship that spans databases -const relationship = crossDatabaseRelationship( - 'read_models.reservations.guest_id', - 'read_models.guests.id', - { onDelete: 'CASCADE' }, -); -``` - -### 5. Migration Dependency Graph (Future) - -Explicit migration dependencies for complex scenarios: - -```typescript -const createUsersMigration = sqlMigration('create_users', [ - /* ... */ -]); -const createPostsMigration = sqlMigration( - 'create_posts', - [ - /* ... */ - ], - { - dependsOn: [createUsersMigration], // Posts table needs users table first - }, -); -``` - -## Open Questions - -### Q1: Function Component Placement - -Should functions be: - -- Top-level components in schema (like tables)? -- Nested within table components? -- Only within feature components? - -**Current decision**: Top-level and within features (flexible) - -### Q2: SQLite Multiple Schema Strategy Default - -For SQLite with multiple schemas, default to: - -- Table prefixing (simpler, single file) -- Separate database files (stronger isolation) -- Error (force explicit choice) - -**Current decision**: Table prefixing (simpler default, user can override) - -### Q3: System Schema Naming Convention - -Should system schema default name be: - -- `__dumbo_system` (clear it's framework) -- `_system` (shorter) -- `system` (clean but might conflict) - -**Current decision**: `system` for dedicated schema, auto-add to first schema otherwise - -### Q4: Sharding Naming Pattern Syntax - -Support only string templates, or also functions? - -- String: `'{shard}_{resource}'` -- Function: `(shard, resource) => ...` - -**Current decision**: Both (string for simplicity, function for flexibility) - -### Q5: Feature Component Registration - -Should there be a registry for custom feature components? - -```typescript -// Register custom feature -registerFeatureComponent('custom_feature', { - detect: (comp) => comp.schemaComponentKey.startsWith('sc:custom:'), - extractMigrations: (comp) => { - /* ... */ - }, -}); -``` - -**Current decision**: Not yet - keep simple, add if needed - -## Implementation Checklist - -### Phase 1: Core Feature Components - -- [ ] `FeatureSchemaComponent` base type -- [ ] `isFeatureSchemaComponent()` type guard -- [ ] `EventStoreSchemaComponent` implementation -- [ ] `PongoCollectionsSchemaComponent` implementation -- [ ] `SystemSchemaComponent` implementation -- [ ] `FunctionSchemaComponent` implementation - -### Phase 2: System Schema Placement - -- [ ] Auto-detection logic for single schema -- [ ] Database type default schema logic (PostgreSQL 'public', etc.) -- [ ] Explicit system schema placement -- [ ] System schema component tests - -### Phase 3: Generic Sharding - -- [ ] `ShardingStrategy` type definition -- [ ] `ShardingConfig` type definition -- [ ] Predefined strategies (`multiTenant`, `region`, `timePartition`) -- [ ] Sharding at database level -- [ ] Sharding at schema level -- [ ] Sharding at table level -- [ ] Exclusion logic for shared resources - -### Phase 4: Database Groups - -- [ ] `DatabaseGroup` type definition -- [ ] `databaseGroup()` factory function -- [ ] Shared system schema across databases -- [ ] Group-level sharding -- [ ] Migration generation for groups - -### Phase 5: Migration Generation - -- [ ] Detect feature components in migration collector -- [ ] Extract internal migrations from features -- [ ] Database-specific migration generation (PostgreSQL vs SQLite) -- [ ] SQLite table prefixing strategy -- [ ] SQLite separate files strategy -- [ ] Sharding-aware migration generation - -### Phase 6: Type Generation - -- [ ] Extract internal structure from features -- [ ] Generate types for event store tables -- [ ] Generate types for Pongo collections -- [ ] Generate types for custom features -- [ ] Exclude system tables from main types (optional) - -### Phase 7: Integration & Documentation - -- [ ] Dumbo schema API updates -- [ ] Pongo schema integration -- [ ] Emmett architecture mapping examples -- [ ] Unit tests for all components -- [ ] Integration tests with PostgreSQL -- [ ] Integration tests with SQLite -- [ ] Documentation and examples - -## Related Files - -- [MIGRATION_UNIFICATION_PLAN.md](./MIGRATION_UNIFICATION_PLAN.md) - Migration system design -- [schemaComponent.ts](./schemaComponent.ts) - Base schema component implementation -- [dumboSchema.ts](./dumboSchema/dumboSchema.ts) - Dumbo schema builder API -- [pongo/schema/index.ts](../../../pongo/src/core/schema/index.ts) - Pongo schema system -- Emmett Architecture (external package) - Component/container/system definitions - -## References - -- [Dumbo Migration Unification Plan](./MIGRATION_UNIFICATION_PLAN.md) - Context on migration system -- [Pongo Strongly Typed Client](https://event-driven.io/en/pongo_strongly_typed_client/) - Pongo collection schema -- [Emmett Projections Testing](https://event-driven.io/en/emmett_projections_testing/) - Event store and projections -- C4 Model - Architectural component hierarchy (system → container → component) - ---- - -**Document Version**: 1.0 -**Last Updated**: 2025-10-31 -**Status**: Design Proposal -**Authors**: Based on collaborative design discussion diff --git a/src/packages/dumbo/src/core/testing/typesTesting.ts b/src/packages/dumbo/src/core/testing/typesTesting.ts index b4a8a44a..3f7d7b5b 100644 --- a/src/packages/dumbo/src/core/testing/typesTesting.ts +++ b/src/packages/dumbo/src/core/testing/typesTesting.ts @@ -1,6 +1,9 @@ +import type { AnyTypeValidationError, TypeValidationSuccess } from '../typing'; + export type Expect = T; -export type Equal = +export type Equals = (() => T extends X ? 1 : 2) extends () => T extends Y ? 1 : 2 ? true : false; -export type IsError = T extends { valid: false } ? true : false; +export type IsError = T extends AnyTypeValidationError ? true : false; +export type IsOK = T extends TypeValidationSuccess ? true : false; diff --git a/src/packages/dumbo/src/core/typing/conditionals.ts b/src/packages/dumbo/src/core/typing/conditionals.ts new file mode 100644 index 00000000..8222e06d --- /dev/null +++ b/src/packages/dumbo/src/core/typing/conditionals.ts @@ -0,0 +1,43 @@ +export type IF = Condition extends true + ? Then + : Else; + +export type OR = A extends true + ? true + : B extends true + ? true + : false; + +export type AND = A extends true + ? B extends true + ? true + : false + : false; + +export type NOT = A extends true ? false : true; + +export type ANY = A extends [infer First, ...infer Rest] + ? First extends true + ? true + : Rest extends boolean[] + ? ANY + : false + : false; + +export type ALL = A extends [infer First, ...infer Rest] + ? First extends true + ? Rest extends boolean[] + ? ALL + : true + : false + : true; + +export type NONE = A extends [infer First, ...infer Rest] + ? First extends true + ? false + : Rest extends boolean[] + ? NONE + : true + : true; + +export type EXTENDS = A extends B ? true : false; diff --git a/src/packages/dumbo/src/core/typing/index.ts b/src/packages/dumbo/src/core/typing/index.ts new file mode 100644 index 00000000..473c79ee --- /dev/null +++ b/src/packages/dumbo/src/core/typing/index.ts @@ -0,0 +1,4 @@ +export * from './conditionals'; +export * from './records'; +export * from './tuples'; +export * from './validation'; diff --git a/src/packages/dumbo/src/core/typing/records.ts b/src/packages/dumbo/src/core/typing/records.ts new file mode 100644 index 00000000..59b3e823 --- /dev/null +++ b/src/packages/dumbo/src/core/typing/records.ts @@ -0,0 +1,4 @@ +export type KeysOfString> = Extract< + keyof T, + string +>; diff --git a/src/packages/dumbo/src/core/typing/tuples.ts b/src/packages/dumbo/src/core/typing/tuples.ts new file mode 100644 index 00000000..077cd725 --- /dev/null +++ b/src/packages/dumbo/src/core/typing/tuples.ts @@ -0,0 +1,148 @@ +import type { AnyTypeValidationError } from './validation'; + +export type GetTupleLength = T['length']; + +export type NotEmptyTuple = + GetTupleLength extends 0 ? never : T; + +export type HaveTuplesTheSameLength< + T extends readonly unknown[], + U extends readonly unknown[], +> = GetTupleLength extends GetTupleLength ? true : false; + +export type IsEmptyTuple = T extends [] + ? true + : false; + +export type IsNotEmptyTuple = + IsEmptyTuple extends true ? false : true; + +export type AllInTuple< + Tuple extends readonly string[], + Union extends string, +> = Tuple extends readonly [infer First, ...infer Rest] + ? First extends Union + ? Rest extends readonly string[] + ? AllInTuple + : true + : false + : true; + +export type FilterExistingInUnion< + Tuple extends readonly string[], + Union extends string, +> = Tuple extends readonly [infer First, ...infer Rest] + ? First extends Union + ? [First, ...FilterNotExistingInUnion] + : [...FilterNotExistingInUnion] + : []; + +export type FilterNotExistingInUnion< + Tuple extends readonly string[], + Union extends string, +> = Tuple extends readonly [infer First, ...infer Rest] + ? First extends Union + ? [ + ...FilterNotExistingInUnion< + Rest extends readonly string[] ? Rest : [], + Union + >, + ] + : [ + First, + ...FilterNotExistingInUnion< + Rest extends readonly string[] ? Rest : [], + Union + >, + ] + : []; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type EnsureTuple = T extends any[] ? T : [T]; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type UnionToIntersection = (U extends any ? (k: U) => void : never) extends ( + k: infer I, +) => void + ? I + : never; + +type LastOfUnion = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + UnionToIntersection T : never> extends () => infer R + ? R + : never; + +type UnionToTuple> = [T] extends [never] + ? [] + : [...UnionToTuple>, L]; + +type TaggedUnion = { [K in keyof T]: [K, T[K]] }[keyof T]; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type ToKeyValue = { + [I in keyof T]: { key: T[I][0]; value: T[I][1] }; +}; + +export type EntriesToTuple = ToKeyValue< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + UnionToTuple> extends [any, any][] + ? UnionToTuple> + : never +>; + +export type ZipTuplesCollectErrors< + TupleA extends readonly unknown[], + TupleB extends readonly unknown[], + ValidateMap, + Accumulated extends AnyTypeValidationError[] = [], +> = [TupleA, TupleB] extends [ + readonly [infer FirstA, ...infer RestA], + readonly [infer FirstB, ...infer RestB], +] + ? FirstA extends keyof ValidateMap + ? FirstB extends keyof ValidateMap[FirstA] + ? ValidateMap[FirstA][FirstB] extends infer Result extends + AnyTypeValidationError + ? ZipTuplesCollectErrors< + RestA, + RestB, + ValidateMap, + [...Accumulated, Result] + > + : ZipTuplesCollectErrors + : ZipTuplesCollectErrors + : ZipTuplesCollectErrors + : Accumulated; + +export type MapEntriesCollectErrors< + Entries extends readonly unknown[], + ValidateMap, + Accumulated extends AnyTypeValidationError[] = [], +> = Entries extends readonly [infer First, ...infer Rest] + ? First extends { key: infer K; value: infer _V } + ? K extends keyof ValidateMap + ? ValidateMap[K] extends infer Result extends AnyTypeValidationError + ? MapEntriesCollectErrors + : MapEntriesCollectErrors + : MapEntriesCollectErrors + : MapEntriesCollectErrors + : Accumulated; + +export type MapRecordCollectErrors< + Record extends object, + ValidateMap, + Accumulated extends AnyTypeValidationError[] = [], + Keys extends readonly unknown[] = UnionToTuple, +> = Keys extends readonly [infer K, ...infer Rest] + ? K extends keyof ValidateMap + ? ValidateMap[K] extends infer Result extends AnyTypeValidationError + ? MapRecordCollectErrors< + Record, + ValidateMap, + [...Accumulated, Result], + Rest + > + : MapRecordCollectErrors + : MapRecordCollectErrors + : Accumulated; diff --git a/src/packages/dumbo/src/core/typing/validation.ts b/src/packages/dumbo/src/core/typing/validation.ts new file mode 100644 index 00000000..ea5fda44 --- /dev/null +++ b/src/packages/dumbo/src/core/typing/validation.ts @@ -0,0 +1,66 @@ +export type TypeValidationResult< + Valid extends boolean = boolean, + Error = never, +> = Valid extends true ? { valid: true } : { valid: false; error: Error }; + +export type TypeValidationError = TypeValidationResult; + +export type TypeValidationSuccess = TypeValidationResult; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyTypeValidationError = TypeValidationError; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type AnyTypeValidationResult = TypeValidationResult; + +export type AnyTypeValidationFailed = + Results extends readonly [infer First, ...infer Rest] + ? First extends { valid: false } + ? true + : Rest extends AnyTypeValidationResult[] + ? AnyTypeValidationFailed + : false + : false; + +export type ExtractTypeValidationErrors = T extends { + valid: false; + error: infer E; +} + ? E + : never; + +export type UnwrapTypeValidationError = + T extends TypeValidationResult ? E : T; + +export type UnwrapTypeValidationErrors< + Results extends readonly AnyTypeValidationResult[], +> = Results extends readonly [infer First, ...infer Rest] + ? First extends TypeValidationResult + ? Rest extends readonly AnyTypeValidationResult[] + ? [E, ...UnwrapTypeValidationErrors] + : [E] + : Rest extends readonly AnyTypeValidationResult[] + ? UnwrapTypeValidationErrors + : [] + : []; + +export type CompareTypes = + Uppercase extends Uppercase ? true : false; + +export type FailOnFirstTypeValidationError< + Validations extends readonly AnyTypeValidationResult[], +> = Validations extends readonly [infer First, ...infer Rest] + ? First extends AnyTypeValidationError + ? First + : Rest extends readonly AnyTypeValidationResult[] + ? FailOnFirstTypeValidationError + : First + : null; + +export type MergeTypeValidationResultIfError< + Errors extends readonly AnyTypeValidationError[], + Result extends AnyTypeValidationResult, +> = Result extends AnyTypeValidationError + ? Errors extends [] + ? [Result] + : [...Errors, Result] + : Errors; diff --git a/todo.md b/todo.md deleted file mode 100644 index 0604cb46..00000000 --- a/todo.md +++ /dev/null @@ -1,78 +0,0 @@ -# Foreign Key Type Validation - Implementation Checklist - -## Phase 1: Foundation - Type Extraction Utilities - -- [x] Step 1: Add ExpectError Helper -- [x] Step 2: Create Foreign Key Types File Structure -- [ ] Step 3: Implement ExtractSchemaNames Type Utility -- [ ] Step 4: Implement ExtractTableNames Type Utility -- [ ] Step 5: Implement ExtractColumnNames Type Utility - -## Phase 2: Column Reference Generation - -- [ ] Step 6: Implement AllColumnReferences Type Utility (Single Schema) -- [ ] Step 7: Test AllColumnReferences with Multi-Schema Database - -## Phase 3: Foreign Key Type Definition - -- [ ] Step 8: Define RelationshipDefinition Type -- [ ] Step 9: Update TableSchemaComponent to Include Foreign Keys -- [ ] Step 10: Update dumboTable to Accept Foreign Keys - -## Phase 4: Single Foreign Key Validation - -- [ ] Step 11: Implement ValidateRelationshipLength -- [ ] Step 12: Implement ValidateRelationshipColumns -- [ ] Step 13: Implement ValidateRelationshipReferences -- [ ] Step 14: Implement ValidateSingleRelationship (Combine Validations) - -## Phase 5: Full Database Validation - -- [ ] Step 15: Implement ValidateTableRelationships -- [ ] Step 16: Implement ValidateSchemaRelationships -- [ ] Step 17: Implement ValidateDatabaseRelationships - -## Phase 6: Integration with dumboSchema - -- [ ] Step 18: Create Foreign Keys Barrel Export -- [ ] Step 19: Wire Validation to database() Function -- [ ] Step 20: Wire Validation to schema() Function - -## Phase 7: Real-World Testing - -- [ ] Step 21: Add Real-World Test Cases -- [ ] Step 22: Update Existing Example in Test File - ---- - -## Quality Gates (Run after EVERY step as subagents) - -After each step is complete, you MUST run: - -1. ✓ `npm run fix` - Fix linting issues -2. ✓ `npm run build:ts` - Ensure TypeScript compiles -3. ✓ `npm run test:unit` - Run all tests - -**All three must pass before proceeding to the next step.** - -If any fail and cannot be fixed automatically, **STOP and ask for help**. - ---- - -## Progress Tracking - -**Current Step:** Not started - -**Completed Steps:** 0 / 22 - -**Estimated Time Remaining:** 4-6 hours - ---- - -## Notes - -- Follow TDD: Write type tests first, then implement -- Use `describe`/`it` for tests -- No unnecessary comments -- Keep code simple and maintainable -- Follow existing codebase patterns