feat(common-sdk): add schema and table enhancements (#86)

Co-authored-by: DominicGBauer <dominic@nomanini.com>
Co-authored-by: Ralf Kistner <ralf@journeyapps.com>

Author: 3 people

demos/angular-supabase-todolist/package.json

@@ -20,7 +20,7 @@
     "@angular/platform-browser-dynamic": "^17.0.4",
     "@angular/router": "^17.0.4",
     "@angular/service-worker": "^17.0.4",
-    "@journeyapps/powersync-sdk-web": "^0.0.3",
+    "@journeyapps/powersync-sdk-web": "workspace:*",
     "@journeyapps/wa-sqlite": "^0.0.2",
     "@supabase/supabase-js": "^2.38.5",
     "rxjs": "~7.8.1",

demos/nextjs-supabase-todolist/src/library/powersync/AppSchema.ts

@@ -1,49 +1,35 @@
-import { Column, ColumnType, Index, IndexedColumn, Schema, Table } from '@journeyapps/powersync-sdk-web';
+import { column, Schema, TableV2 } from '@journeyapps/powersync-sdk-web';
 
-// TODO look into template app composition
+export const LISTS_TABLE = 'lists';
+export const TODOS_TABLE = 'todos';
 
-export interface ListRecord {
-  id: string;
-  name: string;
-  created_at: string;
-  owner_id?: string;
-}
+const todos = new TableV2(
+  {
+    list_id: column.text,
+    created_at: column.text,
+    completed_at: column.text,
+    description: column.text,
+    created_by: column.text,
+    completed_by: column.text,
+    completed: column.integer
+  },
+  { indexes: { list: ['list_id'] } }
+);
 
-export interface TodoRecord {
-  id: string;
-  created_at: string;
-  completed: boolean;
-  description: string;
-  completed_at?: string;
+const lists = new TableV2({
+  created_at: column.text,
+  name: column.text,
+  owner_id: column.text
+});
 
-  created_by: string;
-  completed_by?: string;
-  list_id: string;
-}
+export const AppSchema = new Schema({
+  todos,
+  lists
+});
 
-export const LISTS_TABLE = 'lists';
-export const TODOS_TABLE = 'todos';
+export type Database = (typeof AppSchema)['types'];
+export type TodoRecord = Database['todos'];
+// OR:
+// export type Todo = RowType<typeof todos>;
 
-export const AppSchema = new Schema([
-  new Table({
-    name: TODOS_TABLE,
-    columns: [
-      new Column({ name: 'list_id', type: ColumnType.TEXT }),
-      new Column({ name: 'created_at', type: ColumnType.TEXT }),
-      new Column({ name: 'completed_at', type: ColumnType.TEXT }),
-      new Column({ name: 'description', type: ColumnType.TEXT }),
-      new Column({ name: 'completed', type: ColumnType.INTEGER }),
-      new Column({ name: 'created_by', type: ColumnType.TEXT }),
-      new Column({ name: 'completed_by', type: ColumnType.TEXT })
-    ],
-    indexes: [new Index({ name: 'list', columns: [new IndexedColumn({ name: 'list_id' })] })]
-  }),
-  new Table({
-    name: LISTS_TABLE,
-    columns: [
-      new Column({ name: 'created_at', type: ColumnType.TEXT }),
-      new Column({ name: 'name', type: ColumnType.TEXT }),
-      new Column({ name: 'owner_id', type: ColumnType.TEXT })
-    ]
-  })
-]);
+export type ListRecord = Database['lists'];

demos/yjs-nextjs-supabase-text-collab/src/library/powersync/AppSchema.ts

@@ -1,20 +1,28 @@
-import { Column, ColumnType, Index, IndexedColumn, Schema, Table } from '@journeyapps/powersync-sdk-web';
+import { column, Schema, TableV2 } from '@journeyapps/powersync-sdk-web';
 
-export const AppSchema = new Schema([
-  new Table({
-    name: 'documents',
-    columns: [
-      new Column({ name: 'title', type: ColumnType.TEXT }),
-      new Column({ name: 'created_at', type: ColumnType.TEXT })
-    ]
-  }),
-  new Table({
-    name: 'document_updates',
-    columns: [
-      new Column({ name: 'created_at', type: ColumnType.TEXT }),
-      new Column({ name: 'document_id', type: ColumnType.TEXT }),
-      new Column({ name: 'update_b64', type: ColumnType.TEXT })
-    ],
-    indexes: [new Index({ name: 'by_document', columns: [new IndexedColumn({ name: 'document_id' })] })]
-  })
-]);
+const documents = new TableV2(
+  {
+    title: column.text,
+    created_at: column.text
+  },
+  { indexes: { list: ['list_id'] } }
+);
+
+const document_updates = new TableV2(
+  {
+    document_id: column.text,
+    created_at: column.text,
+    update_b64: column.text
+  },
+  { indexes: { by_document: ['document_id'] } }
+);
+
+export const AppSchema = new Schema({
+  documents,
+  document_updates
+});
+
+export type Database = (typeof AppSchema)['types'];
+export type Documents = Database['documents'];
+
+export type DocumentUpdates = Database['document_updates'];

demos/yjs-nextjs-supabase-text-collab/src/library/powersync/SupabaseConnector.ts

@@ -9,7 +9,6 @@ import {
 import {
   SupabaseClient,
   createClient,
-  PostgrestError,
   FunctionsHttpError,
   FunctionsRelayError,
   FunctionsFetchError

packages/kysely-driver/tests/setup/db.ts

@@ -1,19 +1,12 @@
-import {
-  Column,
-  ColumnType,
-  Schema,
-  Table,
-  WASQLitePowerSyncDatabaseOpenFactory
-} from '@journeyapps/powersync-sdk-web';
+import { Schema, TableV2, WASQLitePowerSyncDatabaseOpenFactory, column } from '@journeyapps/powersync-sdk-web';
 import { wrapPowerSyncWithKysely } from '../../src/sqlite/db';
 import { Database } from './types';
 
-const TestSchema = new Schema([
-  new Table({
-    name: 'users',
-    columns: [new Column({ name: 'name', type: ColumnType.TEXT })]
-  })
-]);
+const users = new TableV2({
+  name: column.text
+});
+
+export const TestSchema = new Schema({ users });
 
 export const getPowerSyncDb = () => {
   const factory = new WASQLitePowerSyncDatabaseOpenFactory({

packages/kysely-driver/tests/setup/types.ts

@@ -1,13 +1,9 @@
-import { ColumnType, Insertable, Selectable, Updateable } from 'kysely';
+import { Insertable, Selectable, Updateable } from 'kysely';
+import { TestSchema } from './db';
 
-export interface Database {
-  users: UsersTable;
-}
+export type Database = (typeof TestSchema)['types'];
 
-export interface UsersTable {
-  id: ColumnType<string, string, never>;
-  name: string;
-}
+export type UsersTable = Database['users'];
 
 export type Users = Selectable<UsersTable>;
 export type NewUsers = Insertable<UsersTable>;

packages/kysely-driver/tests/sqlite/db.test.ts

@@ -1,9 +1,9 @@
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import * as SUT from '../../src/sqlite/db';
 import { Kysely } from 'kysely';
-import { Database } from '../setup/types';
 import { getPowerSyncDb } from '../setup/db';
 import { AbstractPowerSyncDatabase } from '@journeyapps/powersync-sdk-common';
+import { Database } from '../setup/types';
 
 describe('CRUD operations', () => {
   let powerSyncDb: AbstractPowerSyncDatabase;

packages/kysely-driver/tsconfig.json

@@ -5,11 +5,7 @@
     "declaration": true /* Generates corresponding '.d.ts' file. */,
     "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
     "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
-    "lib": [
-      "DOM",
-      "ES2020",
-      "WebWorker"
-    ] /* Specify library files to be included in the compilation. */,
+    "lib": ["DOM", "ES2020", "WebWorker"] /* Specify library files to be included in the compilation. */,
     "module": "es2020" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
     "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */,
     "outDir": "./lib" /* Redirect output structure to the directory. */,
@@ -18,7 +14,10 @@
     "strict": true /* Enable all strict type-checking options. */,
     "target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
   },
-  "include": [
-    "src/**/*",
+  "include": ["src/**/*"],
+  "references": [
+    {
+      "path": "../powersync-sdk-common"
+    }
   ]
 }

packages/powersync-sdk-common/src/client/AbstractPowerSyncDatabase.ts

@@ -114,7 +114,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
     super();
     this.bucketStorageAdapter = this.generateBucketStorageAdapter();
     this.closed = true;
-    this.currentStatus = null;
+    this.currentStatus = undefined;
     this.options = { ...DEFAULT_POWERSYNC_DB_OPTIONS, ...options };
     this._schema = options.schema;
     this.ready = false;
@@ -201,7 +201,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
     try {
       schema.validate();
     } catch (ex) {
-      this.options.logger.warn('Schema validation failed. Unexpected behaviour could occur', ex);
+      this.options.logger?.warn('Schema validation failed. Unexpected behaviour could occur', ex);
     }
     this._schema = schema;
     await this.database.execute('SELECT powersync_replace_schema(?)', [JSON.stringify(this.schema.toJSON())]);
@@ -291,7 +291,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
         [tableGlob]
       );
 
-      if (!existingTableRows.rows.length) {
+      if (!existingTableRows.rows?.length) {
         return;
       }
       for (const row of existingTableRows.rows._array) {
@@ -325,11 +325,11 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
           `SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ${PSInternalTable.CRUD}`
         );
 
-        const row = result.rows.item(0);
+        const row = result.rows!.item(0);
         return new UploadQueueStats(row?.count ?? 0, row?.size ?? 0);
       } else {
         const result = await tx.execute(`SELECT count(*) as count FROM ${PSInternalTable.CRUD}`);
-        const row = result.rows.item(0);
+        const row = result.rows!.item(0);
         return new UploadQueueStats(row?.count ?? 0);
       }
     });
@@ -388,7 +388,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
    * Unlike {@link getCrudBatch}, this only returns data from a single transaction at a time.
    * All data for the transaction is loaded into memory.
    */
-  async getNextCrudTransaction(): Promise<CrudTransaction> {
+  async getNextCrudTransaction(): Promise<CrudTransaction | null> {
     return await this.readTransaction(async (tx) => {
       const first = await tx.getOptional<CrudEntryJSON>(
         `SELECT id, tx_id, data FROM ${PSInternalTable.CRUD} ORDER BY id ASC LIMIT 1`
@@ -543,7 +543,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
 
     const resolvedTables = options?.tables ?? [];
     if (!options?.tables) {
-      const explained = await this.getAll(`EXPLAIN ${sql}`, parameters);
+      const explained = await this.getAll<{ opcode: string; p3: number; p2: number }>(`EXPLAIN ${sql}`, parameters);
       const rootPages = _.chain(explained)
         .filter((row) => row['opcode'] == 'OpenRead' && row['p3'] == 0 && _.isNumber(row['p2']))
         .map((row) => row['p2'])
@@ -571,10 +571,11 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
    * Note, do not declare this as `async *onChange` as it will not work in React Native
    */
   onChange(options?: SQLWatchOptions): AsyncIterable<WatchOnChangeEvent> {
-    const watchedTables = options.tables ?? [];
+    const resolvedOptions = options ?? {};
+    const watchedTables = resolvedOptions.tables ?? [];
 
     let throttledTableUpdates: string[] = [];
-    const throttleMs = options.throttleMs ?? DEFAULT_WATCH_THROTTLE_MS;
+    const throttleMs = resolvedOptions.throttleMs ?? DEFAULT_WATCH_THROTTLE_MS;
 
     return new EventIterator<WatchOnChangeEvent>((eventOptions) => {
       const flushTableUpdates = _.throttle(
@@ -593,7 +594,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
 
       const dispose = this.database.registerListener({
         tablesUpdated: async (update) => {
-          const { rawTableNames } = options;
+          const { rawTableNames } = resolvedOptions;
 
           const tables = isBatchedUpdateNotification(update) ? update.tables : [update.table];
 
@@ -613,7 +614,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
         }
       });
 
-      options.signal?.addEventListener('abort', () => {
+      resolvedOptions.signal?.addEventListener('abort', () => {
         dispose();
         eventOptions.stop();
         // Maybe fail?
@@ -626,7 +627,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
   /**
    * @ignore
    */
-  private async executeReadOnly(sql: string, params: any[]) {
+  private async executeReadOnly(sql: string, params?: any[]) {
     await this.waitForReady();
     return this.database.readLock((tx) => tx.execute(sql, params));
   }

packages/powersync-sdk-common/src/client/AbstractPowerSyncOpenFactory.ts

@@ -33,7 +33,6 @@ export abstract class AbstractPowerSyncDatabaseOpenFactory {
   generateOptions(): PowerSyncDatabaseOptions {
     return {
       database: this.openDB(),
-      schema: this.schema,
       ...this.options
     };
   }