[Fix] Improve bson usage (#172)

stevensJourney · web-flow · commit 62e43aa646a7 · 2024-05-17T13:53:11.000+02:00
diff --git a/.changeset/many-chefs-hug.md b/.changeset/many-chefs-hug.md
@@ -0,0 +1,7 @@
+---
+'@powersync/react-native': minor
+'@powersync/common': minor
+'@powersync/web': minor
+---
+
+Improved import and usage of BSON library.
diff --git a/demos/example-electron/vite.renderer.config.ts b/demos/example-electron/vite.renderer.config.ts
@@ -30,7 +30,6 @@ export default defineConfig((env) => {
         '@powersync/web > js-logger',
         '@powersync/web > lodash/throttle',
         '@powersync/web > can-ndjson-stream',
-        '@powersync/web > bson',
         '@powersync/web > buffer',
         '@powersync/web > rsocket-core',
         '@powersync/web > rsocket-websocket-client',
diff --git a/demos/example-nextjs/package.json b/demos/example-nextjs/package.json
@@ -30,8 +30,8 @@
     "lowlight": "^2.9.0",
     "next": "14.1.0",
     "next-images": "1.8.5",
-    "react": "^18.2.0",
-    "react-dom": "^18.2.0",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
     "remixicon": "^2.5.0",
     "shiki": "^0.10.1",
     "simplify-js": "^1.2.4"
diff --git a/demos/example-vite/src/index.js b/demos/example-vite/src/index.js
@@ -1,4 +1,7 @@
 import { Column, ColumnType, WASQLitePowerSyncDatabaseOpenFactory, Schema, Table } from '@powersync/web';
+import Logger from 'js-logger';
+
+Logger.useDefaults();
 
 /**
  * A placeholder connector which doesn't do anything.
diff --git a/demos/example-vite/vite.config.ts b/demos/example-vite/vite.config.ts
@@ -1,6 +1,5 @@
 import wasm from 'vite-plugin-wasm';
 import topLevelAwait from 'vite-plugin-top-level-await';
-
 import { defineConfig } from 'vite';
 
 // https://vitejs.dev/config/
@@ -13,12 +12,6 @@ export default defineConfig({
     },
     emptyOutDir: true
   },
-  resolve: {
-    alias: [
-      // https://jira.mongodb.org/browse/NODE-5773
-      { find: 'bson', replacement: require.resolve('bson') }
-    ]
-  },
   envDir: '..', // Use this dir for env vars, not 'src'.
   optimizeDeps: {
     // Don't optimize these packages as they contain web workers and WASM files.
@@ -29,7 +22,6 @@ export default defineConfig({
       '@powersync/web > js-logger',
       '@powersync/web > lodash/throttle',
       '@powersync/web > can-ndjson-stream',
-      '@powersync/web > bson',
       '@powersync/web > buffer',
       '@powersync/web > rsocket-core',
       '@powersync/web > rsocket-websocket-client',
diff --git a/demos/example-webpack/package.json b/demos/example-webpack/package.json
@@ -11,10 +11,10 @@
     "@powersync/web": "workspace:*"
   },
   "devDependencies": {
-    "webpack-cli": "^5.1.4",
-    "webpack": "^5.90.1",
     "@types/webpack": "^5.28.5",
+    "html-webpack-plugin": "^5.6.0",
     "serve": "^14.2.1",
-    "html-webpack-plugin": "^5.6.0"
+    "webpack": "^5.90.1",
+    "webpack-cli": "^5.1.4"
   }
 }
diff --git a/demos/example-webpack/src/index.js b/demos/example-webpack/src/index.js
@@ -1,4 +1,7 @@
 import { Column, ColumnType, WASQLitePowerSyncDatabaseOpenFactory, Schema, Table } from '@powersync/web';
+import Logger from 'js-logger';
+
+Logger.useDefaults();
 
 /**
  * A placeholder connector which doesn't do anything.
diff --git a/demos/example-webpack/webpack.config.js b/demos/example-webpack/webpack.config.js
@@ -7,6 +7,12 @@ module.exports = {
     filename: 'index.js',
     path: path.join(__dirname, 'dist')
   },
+  externals: {
+    // BSON includes imports to these, but does not have a hard requirement for them to be present.
+    crypto: 'Crypto',
+    stream: 'Stream',
+    vm: 'VM'
+  },
   devtool: 'source-map',
   mode: 'development',
   resolve: {
diff --git a/demos/react-supabase-todolist/package.json b/demos/react-supabase-todolist/package.json
@@ -11,12 +11,15 @@
   "dependencies": {
     "@powersync/react": "workspace:*",
     "@powersync/web": "workspace:*",
+    "@emotion/react": "11.11.4",
+    "@emotion/styled": "11.11.5",
     "@journeyapps/wa-sqlite": "~0.2.0",
     "@mui/icons-material": "^5.15.12",
     "@mui/material": "^5.15.12",
     "@mui/x-data-grid": "^6.19.6",
     "@supabase/supabase-js": "^2.39.7",
     "buffer": "^6.0.3",
+    "formik": "^2.4.6",
     "js-logger": "^1.6.1",
     "lodash": "^4.17.21",
     "react": "^18.2.0",
diff --git a/demos/react-supabase-todolist/vite.config.mts b/demos/react-supabase-todolist/vite.config.mts
@@ -5,8 +5,6 @@ import { fileURLToPath, URL } from 'url';
 import { defineConfig } from 'vite';
 import react from '@vitejs/plugin-react';
 import { VitePWA } from 'vite-plugin-pwa';
-import { createRequire } from 'node:module';
-const require = createRequire(import.meta.url); // Needed since the config file is also an ES module
 
 // https://vitejs.dev/config/
 export default defineConfig({
@@ -19,11 +17,7 @@ export default defineConfig({
     emptyOutDir: true
   },
   resolve: {
-    alias: [
-      { find: '@', replacement: fileURLToPath(new URL('./src', import.meta.url)) },
-      // https://jira.mongodb.org/browse/NODE-5773
-      { find: 'bson', replacement: require.resolve('bson') }
-    ]
+    alias: [{ find: '@', replacement: fileURLToPath(new URL('./src', import.meta.url)) }]
   },
   publicDir: '../public',
   envDir: '..', // Use this dir for env vars, not 'src'.
@@ -36,7 +30,6 @@ export default defineConfig({
       '@powersync/web > js-logger',
       '@powersync/web > lodash/throttle',
       '@powersync/web > can-ndjson-stream',
-      '@powersync/web > bson',
       '@powersync/web > buffer',
       '@powersync/web > rsocket-core',
       '@powersync/web > rsocket-websocket-client',
diff --git a/demos/vue-supabase-todolist/vite.config.ts b/demos/vue-supabase-todolist/vite.config.ts
@@ -70,11 +70,7 @@ export default defineConfig({
   ],
   define: { 'process.env': {} },
   resolve: {
-    alias: [
-      { find: '@', replacement: fileURLToPath(new URL('./src', import.meta.url)) },
-      // https://jira.mongodb.org/browse/NODE-5773
-      { find: 'bson', replacement: require.resolve('bson') }
-    ],
+    alias: [{ find: '@', replacement: fileURLToPath(new URL('./src', import.meta.url)) }],
     extensions: ['.js', '.json', '.jsx', '.mjs', '.ts', '.tsx', '.vue']
   },
   optimizeDeps: {
@@ -86,7 +82,6 @@ export default defineConfig({
       '@powersync/web > js-logger',
       '@powersync/web > lodash/throttle',
       '@powersync/web > can-ndjson-stream',
-      '@powersync/web > bson',
       '@powersync/web > buffer',
       '@powersync/web > rsocket-core',
       '@powersync/web > rsocket-websocket-client',
diff --git a/demos/yjs-react-supabase-text-collab/vite.config.mts b/demos/yjs-react-supabase-text-collab/vite.config.mts
@@ -38,7 +38,6 @@ export default defineConfig({
       '@powersync/web > js-logger',
       '@powersync/web > lodash/throttle',
       '@powersync/web > can-ndjson-stream',
-      '@powersync/web > bson',
       '@powersync/web > buffer',
       '@powersync/web > rsocket-core',
       '@powersync/web > rsocket-websocket-client',
diff --git a/packages/common/package.json b/packages/common/package.json
@@ -28,7 +28,6 @@
   },
   "dependencies": {
     "async-mutex": "^0.4.0",
-    "bson": "^6.6.0",
     "buffer": "^6.0.3",
     "can-ndjson-stream": "^1.0.2",
     "cross-fetch": "^4.0.0",
@@ -43,6 +42,7 @@
     "@types/node": "^20.5.9",
     "@types/uuid": "^9.0.1",
     "typescript": "^5.1.3",
-    "vitest": "^1.5.2"
+    "vitest": "^1.5.2",
+    "bson": "^6.6.0"
   }
 }
diff --git a/packages/common/src/client/sync/stream/AbstractRemote.ts b/packages/common/src/client/sync/stream/AbstractRemote.ts
@@ -6,10 +6,12 @@ import { DataStream } from '../../../utils/DataStream';
 import ndjsonStream from 'can-ndjson-stream';
 import { RSocket, RSocketConnector, Requestable } from 'rsocket-core';
 import { WebsocketClientTransport } from 'rsocket-websocket-client';
-import { serialize, deserialize } from 'bson';
+import type { BSON } from 'bson';
 import { AbortOperation } from '../../../utils/AbortOperation';
 import { Buffer } from 'buffer';
 
+export type BSONImplementation = typeof BSON;
+
 export type RemoteConnector = {
   fetchCredentials: () => Promise<PowerSyncCredentials | null>;
 };
@@ -171,13 +173,20 @@ export abstract class AbstractRemote {
     return res;
   }
 
+  /**
+   * Provides a BSON implementation. The import nature of this varies depending on the platform
+   */
+  abstract getBSON(): Promise<BSONImplementation>;
+
   /**
    * Connects to the sync/stream websocket endpoint
    */
   async socketStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>> {
     const { path } = options;
     const request = await this.buildRequest(path);
 
+    const bson = await this.getBSON();
+
     const connector = new RSocketConnector({
       transport: new WebsocketClientTransport({
         url: this.options.socketUrlTransformer(request.url)
@@ -190,7 +199,7 @@ export abstract class AbstractRemote {
         payload: {
           data: null,
           metadata: Buffer.from(
-            serialize({
+            bson.serialize({
               token: request.headers.Authorization
             })
           )
@@ -223,7 +232,7 @@ export abstract class AbstractRemote {
       }
       socketIsClosed = true;
       rsocket.close();
-    }
+    };
     // We initially request this amount and expect these to arrive eventually
     let pendingEventsCount = SYNC_QUEUE_REQUEST_N;
 
@@ -232,9 +241,9 @@ export abstract class AbstractRemote {
 
       const res = rsocket.requestStream(
         {
-          data: Buffer.from(serialize(options.data)),
+          data: Buffer.from(bson.serialize(options.data)),
           metadata: Buffer.from(
-            serialize({
+            bson.serialize({
               path
             })
           )
@@ -268,7 +277,7 @@ export abstract class AbstractRemote {
               return;
             }
 
-            const deserializedData = deserialize(data);
+            const deserializedData = bson.deserialize(data);
             stream.enqueueData(deserializedData);
           },
           onComplete: () => {
diff --git a/packages/react-native/package.json b/packages/react-native/package.json
@@ -35,7 +35,8 @@
   },
   "dependencies": {
     "@powersync/react": "workspace:*",
-    "async-lock": "^1.4.0"
+    "async-lock": "^1.4.0",
+    "bson": "^6.6.0"
   },
   "devDependencies": {
     "@journeyapps/react-native-quick-sqlite": "^1.1.6",
diff --git a/packages/react-native/src/sync/stream/ReactNativeRemote.ts b/packages/react-native/src/sync/stream/ReactNativeRemote.ts
@@ -1,6 +1,13 @@
-import { AbstractRemote, DataStream, StreamingSyncLine, SyncStreamOptions } from '@powersync/common';
+import {
+  AbstractRemote,
+  BSONImplementation,
+  DataStream,
+  StreamingSyncLine,
+  SyncStreamOptions
+} from '@powersync/common';
 import { Platform } from 'react-native';
-
+// Note docs for React Native https://github.com/mongodb/js-bson?tab=readme-ov-file#react-native
+import { BSON } from 'bson';
 export const STREAMING_POST_TIMEOUT_MS = 30_000;
 
 type PolyfillTest = {
@@ -53,6 +60,10 @@ ${missingPolyfills.join('\n')}`
 };
 
 export class ReactNativeRemote extends AbstractRemote {
+  async getBSON(): Promise<BSONImplementation> {
+    return BSON;
+  }
+
   async socketStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>> {
     validatePolyfills(SocketPolyfillTests);
     return super.socketStream(options);
diff --git a/packages/web/README.md b/packages/web/README.md
@@ -56,6 +56,14 @@ if (typeof self.Buffer == 'undefined') {
 }
 ```
 
+## Webpack
+
+See the [example Webpack config](https://github.com/powersync-ja/powersync-js/blob/main/demos/example-webpack/webpack.config.js) for details on polyfills and requirements.
+
+## Vite
+
+See the [example Vite config](https://github.com/powersync-ja/powersync-js/blob/main/demos/example-vite/vite.config.ts) for details on polyfills and requirements.
+
 # Getting Started
 
 Our [full SDK reference](https://docs.powersync.com/client-sdk-references/js-web) contains everything you need to know to get started implementing PowerSync in your project.
diff --git a/packages/web/package.json b/packages/web/package.json
@@ -39,6 +39,7 @@
   "dependencies": {
     "async-mutex": "^0.4.0",
     "buffer": "^6.0.3",
+    "bson": "^6.6.0",
     "comlink": "^4.4.1",
     "js-logger": "^1.6.1",
     "lodash": "^4.17.21"
diff --git a/packages/web/src/db/sync/WebRemote.ts b/packages/web/src/db/sync/WebRemote.ts
@@ -1,3 +1,18 @@
-import { AbstractRemote } from '@powersync/common';
+import { AbstractRemote, BSONImplementation } from '@powersync/common';
 
-export class WebRemote extends AbstractRemote {}
+export class WebRemote extends AbstractRemote {
+  private _bson: BSONImplementation | undefined;
+
+  async getBSON(): Promise<BSONImplementation> {
+    if (this._bson) {
+      return this._bson;
+    }
+
+    /**
+     * Dynamic import to be used only when needed.
+     */
+    const { BSON } = await import('bson');
+    this._bson = BSON;
+    return this._bson;
+  }
+}
diff --git a/packages/web/tests/utils/MockStreamOpenFactory.ts b/packages/web/tests/utils/MockStreamOpenFactory.ts
@@ -8,7 +8,8 @@ import {
   PowerSyncDatabaseOptions,
   SyncStreamOptions,
   DataStream,
-  StreamingSyncLine
+  StreamingSyncLine,
+  BSONImplementation
 } from '@powersync/common';
 import {
   PowerSyncDatabase,
@@ -41,6 +42,10 @@ export class MockRemote extends AbstractRemote {
     this.streamController = null;
   }
 
+  async getBSON(): Promise<BSONImplementation> {
+    return import('bson');
+  }
+
   post(path: string, data: any, headers?: Record<string, string> | undefined): Promise<any> {
     throw new Error('Method not implemented.');
   }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/tools/diagnostics-app/vite.config.mts b/tools/diagnostics-app/vite.config.mts

Original file line number	Diff line number	Diff line change
`@@ -11,10 +11,10 @@`
`11`	`11`	`"@powersync/web": "workspace:*"`
`12`	`12`	`},`
`13`	`13`	`"devDependencies": {`
`14`		`- "webpack-cli": "^5.1.4",`
`15`		`- "webpack": "^5.90.1",`
`16`	`14`	`"@types/webpack": "^5.28.5",`
	`15`	`+ "html-webpack-plugin": "^5.6.0",`
`17`	`16`	`"serve": "^14.2.1",`
`18`		`- "html-webpack-plugin": "^5.6.0"`
	`17`	`+ "webpack": "^5.90.1",`
	`18`	`+ "webpack-cli": "^5.1.4"`
`19`	`19`	`}`
`20`	`20`	`}`