Sam/remove tx metadata files

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- added: Remove empty transaction metadata files when loading transaction data.
+- changed: Use syncRepo from upgraded edge-sync-client for syncing repo algorithm.
+
 ## 2.36.0 (2025-11-04)
 
 - added: Added `EdgeSubscribedAddress` type for `onSubscribeAddresses` and `startEngine`.

package.json

@@ -86,7 +86,7 @@
     "cleaners": "^0.3.17",
     "currency-codes": "^1.5.1",
     "disklet": "^0.5.2",
-    "edge-sync-client": "^0.2.8",
+    "edge-sync-client": "../edge-sync-client",
     "elliptic": "^6.4.0",
     "hash.js": "^1.1.7",
     "hmac-drbg": "^1.0.1",

src/core/actions.ts

@@ -319,6 +319,14 @@ export type RootAction =
         walletId: string
       }
     }
+  | {
+      // Called when empty transaction metadata files have been deleted.
+      type: 'CURRENCY_WALLET_FILE_DELETED'
+      payload: {
+        txidHashes: string[]
+        walletId: string
+      }
+    }
   | {
       type: 'CURRENCY_WALLET_LOADED_TOKEN_FILE'
       payload: {

src/core/context/internal-api.ts

@@ -1,4 +1,5 @@
 import { Disklet } from 'disklet'
+import { SyncResult } from 'edge-sync-client'
 import { Bridgeable, bridgifyObject, close, emit, update } from 'yaob'
 import { Unsubscribe } from 'yavent'
 
@@ -13,7 +14,7 @@ import {
 import { loginFetch } from '../login/login-fetch'
 import { hashUsername } from '../login/login-selectors'
 import { ApiInput } from '../root-pixie'
-import { makeRepoPaths, syncRepo, SyncResult } from '../storage/repo'
+import { makeRepoPaths, syncRepo } from '../storage/repo'
 
 /**
  * The requesting side of an Edge login lobby.
@@ -98,13 +99,10 @@ export class EdgeInternalStuff extends Bridgeable<{}> {
     await sendLobbyReply(this._ai, lobbyId, lobbyRequest, replyData)
   }
 
-  async syncRepo(syncKey: Uint8Array): Promise<SyncResult> {
+  syncRepo(syncKey: Uint8Array): Promise<SyncResult> {
     const { io, syncClient } = this._ai.props
     const paths = makeRepoPaths(io, { dataKey: new Uint8Array(0), syncKey })
-    return await syncRepo(syncClient, paths, {
-      lastSync: 0,
-      lastHash: undefined
-    })
+    return syncRepo(syncClient, paths, { lastSync: 0, lastHash: undefined })
   }
 
   async getRepoDisklet(

src/core/currency/wallet/currency-wallet-files.ts

@@ -37,7 +37,7 @@ import {
 import { CurrencyWalletInput } from './currency-wallet-pixie'
 import { TxFileNames } from './currency-wallet-reducer'
 import { currencyCodesToTokenIds } from './enabled-tokens'
-import { mergeMetadata } from './metadata'
+import { isEmptyMetadata, mergeMetadata } from './metadata'
 
 const CURRENCY_FILE = 'Currency.json'
 const LEGACY_MAP_FILE = 'fixedLegacyFileNames.json'
@@ -46,6 +46,34 @@ const SEEN_TX_CHECKPOINT_FILE = 'seenTxCheckpoint.json'
 const TOKENS_FILE = 'Tokens.json'
 const WALLET_NAME_FILE = 'WalletName.json'
 
+/**
+ * Checks if a transaction file is "empty" (contains no user metadata).
+ * An empty file is one that matches the default template with no user-added data.
+ */
+export function isEmptyTxFile(file: TransactionFile): boolean {
+  // Check top-level user data fields:
+  if (file.savedAction != null) return false
+  if (file.swap != null) return false
+  if (file.payees != null && file.payees.length > 0) return false
+  if (file.deviceDescription != null) return false
+  if (file.secret != null) return false
+  if (file.feeRateRequested != null) return false
+
+  // Check currencies map for non-empty metadata:
+  for (const asset of file.currencies.values()) {
+    if (!isEmptyMetadata(asset.metadata)) return false
+    if (asset.assetAction != null) return false
+  }
+
+  // Check tokens map for non-empty metadata:
+  for (const asset of file.tokens.values()) {
+    if (!isEmptyMetadata(asset.metadata)) return false
+    if (asset.assetAction != null) return false
+  }
+
+  return true
+}
+
 const legacyAddressFile = makeJsonFile(asLegacyAddressFile)
 const legacyMapFile = makeJsonFile(asLegacyMapFile)
 const legacyTokensFile = makeJsonFile(asLegacyTokensFile)
@@ -297,7 +325,10 @@ export async function loadTxFiles(
   const fileNames = input.props.walletState.fileNames
   const walletFiat = input.props.walletState.fiat
 
-  const out: { [filename: string]: TransactionFile } = {}
+  const out: { [txidHash: string]: TransactionFile } = {}
+  const emptyFileInfos: Array<{ txidHash: string; path: string }> = []
+
+  // Load legacy transaction files:
   await Promise.all(
     txIdHashes.map(async txidHash => {
       if (fileNames[txidHash] == null) return
@@ -307,6 +338,8 @@ export async function loadTxFiles(
       out[txidHash] = fixLegacyFile(clean, walletCurrency, walletFiat)
     })
   )
+
+  // Load new transaction files:
   await Promise.all(
     txIdHashes.map(async txidHash => {
       if (fileNames[txidHash] == null) return
@@ -317,6 +350,38 @@ export async function loadTxFiles(
     })
   )
 
+  // Detect empty files and queue them for deletion:
+  for (const txidHash of Object.keys(out)) {
+    const file = out[txidHash]
+    if (isEmptyTxFile(file)) {
+      const path = `transaction/${fileNames[txidHash].fileName}`
+      emptyFileInfos.push({ txidHash, path })
+      // Remove from output so it's not loaded into state:
+      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+      delete out[txidHash]
+    }
+  }
+
+  // Delete empty files in a non-blocking way (fire-and-forget):
+  if (emptyFileInfos.length > 0) {
+    const txidHashes: string[] = []
+    for (const info of emptyFileInfos) {
+      txidHashes.push(info.txidHash)
+      // Delete files without awaiting:
+      disklet.delete(info.path).catch(error => {
+        input.props.log.warn(
+          `Failed to delete empty tx file ${info.path}: ${String(error)}`
+        )
+      })
+    }
+    // Dispatch action to remove from fileNames state so that loadTxFiles
+    // won't attempt to load these empty files again:
+    dispatch({
+      type: 'CURRENCY_WALLET_FILE_DELETED',
+      payload: { txidHashes, walletId }
+    })
+  }
+
   dispatch({
     type: 'CURRENCY_WALLET_FILES_LOADED',
     payload: { files: out, walletId }

src/core/currency/wallet/currency-wallet-reducer.ts

@@ -286,6 +286,15 @@ const currencyWalletInner = buildReducer<
           ...files
         }
       }
+      case 'CURRENCY_WALLET_FILE_DELETED': {
+        const { txidHashes } = action.payload
+        const out = { ...state }
+        for (const txidHash of txidHashes) {
+          // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+          delete out[txidHash]
+        }
+        return out
+      }
     }
     return state
   },
@@ -309,6 +318,15 @@ const currencyWalletInner = buildReducer<
         }
         return state
       }
+      case 'CURRENCY_WALLET_FILE_DELETED': {
+        const { txidHashes } = action.payload
+        const out = { ...state }
+        for (const txidHash of txidHashes) {
+          // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+          delete out[txidHash]
+        }
+        return out
+      }
     }
     return state
   },

src/core/currency/wallet/metadata.ts

@@ -44,6 +44,23 @@ export function mergeMetadata(
   return out
 }
 
+/**
+ * Checks if metadata is empty (contains no user-added data).
+ */
+export function isEmptyMetadata(metadata: EdgeMetadata): boolean {
+  if (metadata.bizId != null) return false
+  if (metadata.category != null && metadata.category !== '') return false
+  if (metadata.name != null && metadata.name !== '') return false
+  if (metadata.notes != null && metadata.notes !== '') return false
+  if (
+    metadata.exchangeAmount != null &&
+    Object.keys(metadata.exchangeAmount).length > 0
+  ) {
+    return false
+  }
+  return true
+}
+
 const asDiskMetadata = asObject({
   bizId: asOptional(asNumber),
   category: asOptional(asString),

src/core/storage/encrypt-disklet.ts

@@ -6,14 +6,30 @@ import { EdgeIo } from '../../types/types'
 import { decrypt, decryptText, encrypt } from '../../util/crypto/crypto'
 import { utf8 } from '../../util/encoding'
 
+/**
+ * Creates an encrypted disklet that wraps another disklet.
+ * Optionally accepts a deletedDisklet for sync-aware deletions.
+ * When deletedDisklet is provided, delete operations will mark files
+ * for deletion by writing an empty file to the deleted/ directory,
+ * which will be processed during the next sync.
+ */
 export function encryptDisklet(
   io: EdgeIo,
   dataKey: Uint8Array,
-  disklet: Disklet
+  disklet: Disklet,
+  /** Provide when this disklet is synchronized with edge-sync-client's syncRepo */
+  deletedDisklet?: Disklet
 ): Disklet {
   const out = {
-    delete(path: string): Promise<unknown> {
-      return disklet.delete(path)
+    async delete(path: string): Promise<unknown> {
+      // If we have a deletedDisklet, mark the file for deletion
+      // by writing an empty file to the deleted/ directory.
+      // The sync process will handle the actual deletion.
+      if (deletedDisklet != null) {
+        await deletedDisklet.setText(path, '')
+      }
+      // Also delete locally for immediate effect:
+      return await disklet.delete(path)
     },
 
     async getData(path: string): Promise<Uint8Array> {

src/core/storage/repo.ts

@@ -1,9 +1,8 @@
 import { Disklet, mergeDisklets, navigateDisklet } from 'disklet'
-import type { EdgeBox as SyncEdgeBox } from 'edge-sync-client'
-import { SyncClient } from 'edge-sync-client'
+import { SyncClient, SyncResult } from 'edge-sync-client'
 import { base16, base64 } from 'rfc4648'
 
-import { asEdgeBox, wasEdgeBox } from '../../types/server-cleaners'
+import { wasEdgeBox } from '../../types/server-cleaners'
 import { EdgeBox } from '../../types/server-types'
 import { EdgeIo } from '../../types/types'
 import { sha256 } from '../../util/crypto/hashes'
@@ -12,17 +11,10 @@ import { EdgeStorageKeys } from '../login/storage-keys'
 import { encryptDisklet } from './encrypt-disklet'
 import { StorageWalletPaths, StorageWalletStatus } from './storage-reducer'
 
-const CHANGESET_MAX_ENTRIES = 100
-
 interface RepoChanges {
   [path: string]: EdgeBox | null
 }
 
-export interface SyncResult {
-  changes: RepoChanges
-  status: StorageWalletStatus
-}
-
 export function makeLocalDisklet(io: EdgeIo, walletId: string): Disklet {
   return navigateDisklet(
     io.disklet,
@@ -45,10 +37,12 @@ export function makeRepoPaths(
   )
   const changesDisklet = navigateDisklet(baseDisklet, 'changes')
   const dataDisklet = navigateDisklet(baseDisklet, 'data')
+  const deletedDisklet = navigateDisklet(baseDisklet, 'deleted')
   const disklet = encryptDisklet(
     io,
     dataKey,
-    mergeDisklets(changesDisklet, dataDisklet)
+    mergeDisklets(changesDisklet, dataDisklet),
+    deletedDisklet
   )
 
   return {
@@ -58,6 +52,7 @@ export function makeRepoPaths(
     baseDisklet,
     changesDisklet,
     dataDisklet,
+    deletedDisklet,
     disklet
   }
 }
@@ -99,81 +94,10 @@ export async function syncRepo(
   paths: StorageWalletPaths,
   status: StorageWalletStatus
 ): Promise<SyncResult> {
-  const { changesDisklet, dataDisklet, syncKey } = paths
-
-  const ourChanges: Array<{
-    path: string
-    box: EdgeBox
-  }> = await deepListWithLimit(changesDisklet).then(paths => {
-    return Promise.all(
-      paths.map(async path => ({
-        path,
-        box: asEdgeBox(JSON.parse(await changesDisklet.getText(path)))
-      }))
-    )
-  })
-
-  const syncKeyEncoded = base16.stringify(syncKey).toLowerCase()
-
-  // Send a read request if no changes present locally, otherwise bundle the
-  // changes with the a update request.
-  const reply = await (() => {
-    // Read the repo if no changes present locally.
-    if (ourChanges.length === 0) {
-      return syncClient.readRepo(syncKeyEncoded, status.lastHash)
-    }
-
-    // Write local changes to the repo.
-    const changes: { [name: string]: SyncEdgeBox } = {}
-    for (const change of ourChanges) {
-      changes[change.path] = wasEdgeBox(change.box)
-    }
-    return syncClient.updateRepo(syncKeyEncoded, status.lastHash, { changes })
-  })()
-
-  // Make the request:
-  const { hash } = reply
-  const changes: RepoChanges = {}
-  for (const path of Object.keys(reply.changes ?? {})) {
-    const json = reply.changes[path]
-    changes[path] = json == null ? null : asEdgeBox(json)
-  }
-
-  // Save the incoming changes into our `data` folder:
-  await saveChanges(dataDisklet, changes)
-
-  // Delete any changed keys (since the upload is done):
-  await Promise.all(
-    ourChanges.map(change => changesDisklet.delete(change.path))
+  const syncKeyEncoded = base16.stringify(paths.syncKey).toLowerCase()
+  return await syncClient.syncRepo(
+    paths.baseDisklet,
+    syncKeyEncoded,
+    status.lastHash
   )
-
-  // Update the repo status:
-  status.lastSync = Date.now() / 1000
-  if (hash != null) status.lastHash = hash
-  await paths.baseDisklet.setText('status.json', JSON.stringify(status))
-  return { status, changes }
-}
-
-/**
- * Lists all files in a disklet, recursively up to a limit.
- * Returns a list of full paths.
- */
-async function deepListWithLimit(
-  disklet: Disklet,
-  path: string = '',
-  limit: number = CHANGESET_MAX_ENTRIES
-): Promise<string[]> {
-  const list = await disklet.list(path)
-  const paths = Object.keys(list).filter(path => list[path] === 'file')
-  const folders = Object.keys(list).filter(path => list[path] === 'folder')
-
-  // Loop over folders to get subpaths
-  for (const folder of folders) {
-    if (paths.length >= limit) break
-    const remaining = limit - paths.length
-    const subpaths = await deepListWithLimit(disklet, folder, remaining)
-    paths.push(...subpaths.slice(0, remaining))
-  }
-
-  return paths
 }