From 133f9fd0c2c7c70edeb7cdac95e2f4c5511c5971 Mon Sep 17 00:00:00 2001 From: Peter Banik Date: Sat, 28 Mar 2026 02:35:30 +0100 Subject: [PATCH] feat: add ESEARCH (RFC 4731) and PARTIAL (RFC 9394) support Extend search() to accept returnOptions for efficient server-side result summarization. When the server advertises ESEARCH capability, emits UID SEARCH RETURN (...) and parses the ESEARCH untagged response into an ESearchResult object (count, min, max, all, partial). Key changes: - ESEARCH response parser (parseEsearchResponse) in search.js - RETURN clause builder with PARTIAL range support - Client-side ESearchResult derivation when server lacks ESEARCH - TypeScript ESearchResult interface and search() overloads - 13 nodeunit tests (parser, command builder, fallback, compat) Backward compatible: callers without returnOptions still receive number[] exactly as before. --- lib/commands/search.js | 132 +++++++++++++++++++++++- lib/imap-flow.d.ts | 26 ++++- lib/imap-flow.js | 37 ++++++- test/search-test.js | 228 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 420 insertions(+), 3 deletions(-) create mode 100644 test/search-test.js diff --git a/lib/commands/search.js b/lib/commands/search.js index feab28d..06555f6 100644 --- a/lib/commands/search.js +++ b/lib/commands/search.js @@ -3,6 +3,70 @@ const { enhanceCommandError } = require('../tools.js'); const { searchCompiler } = require('../search-compiler.js'); +/** + * Parses the key-value attributes from an ESEARCH untagged response. + * + * Receives the attribute list AFTER stripping the leading (TAG "X") list + * and the UID atom — i.e. only the result keyword/value pairs remain. + * + * ALL and PARTIAL.messages are kept as compact sequence-set strings. + * Use expandRange() from tools.js if you need to expand them. + * + * @param {Array} attrs - Attribute array from the IMAP parser + * @returns {Object} ESearchResult object + */ +function parseEsearchResponse(attrs) { + const result = {}; + let i = 0; + while (i < attrs.length) { + const token = attrs[i]; + if (!token || token.type !== 'ATOM') { i++; continue; } + const key = token.value.toUpperCase(); + if (i + 1 >= attrs.length) { i++; continue; } + switch (key) { + case 'COUNT': { + const n = Number(attrs[++i]?.value); + if (!isNaN(n)) result.count = n; + break; + } + case 'MIN': { + const n = Number(attrs[++i]?.value); + if (!isNaN(n)) result.min = n; + break; + } + case 'MAX': { + const n = Number(attrs[++i]?.value); + if (!isNaN(n)) result.max = n; + break; + } + case 'ALL': { + const allToken = attrs[++i]; + if (allToken && typeof allToken.value === 'string') { + result.all = allToken.value; + } + break; + } + case 'PARTIAL': { + const listToken = attrs[++i]; + if (!listToken || !Array.isArray(listToken.attributes) || listToken.attributes.length < 2) break; + result.partial = { + range: listToken.attributes[0].value, + messages: listToken.attributes[1].value + }; + break; + } + default: + // Skip the value token for unknown keys to keep the stream aligned. + // The loop's unconditional i++ at the bottom advances past the key; + // this extra i++ advances past the value token. + i++; + break; + } + i++; + } + return result; +} + /** * Searches for messages matching the specified criteria. * @@ -10,7 +74,11 @@ const { searchCompiler } = require('../search-compiler.js'); * @param {Object|boolean} query - Search query object, or true/empty object to match all messages * @param {Object} [options] - Search options * @param {boolean} [options.uid] - If true, use UID SEARCH instead of SEARCH - * @returns {Promise} Sorted array of matching sequence numbers or UIDs, or false on failure + * @param {Array} [options.returnOptions] - ESEARCH RETURN options. When present AND the + * server advertises ESEARCH capability, triggers ESEARCH and returns an ESearchResult. + * Items are strings ('MIN','MAX','COUNT','ALL') or objects ({ partial: '1:100' }). + * When server lacks ESEARCH, falls back to plain SEARCH and returns number[]. + * @returns {Promise} */ module.exports = async (connection, query, options) => { if (connection.state !== connection.states.SELECTED) { @@ -36,6 +104,65 @@ module.exports = async (connection, query, options) => { return false; } + const useEsearch = options.returnOptions && options.returnOptions.length > 0 && connection.capabilities.has('ESEARCH'); + + if (useEsearch) { + // Build RETURN (...) item list + const returnItems = []; + for (const opt of options.returnOptions) { + if (typeof opt === 'string') { + returnItems.push({ type: 'ATOM', value: opt.toUpperCase() }); + } else if (opt && typeof opt.partial === 'string') { + // RFC 9394: PARTIAL is an atom followed by the range atom, both inside RETURN (...) + returnItems.push({ type: 'ATOM', value: 'PARTIAL' }); + returnItems.push({ type: 'ATOM', value: opt.partial }); + } + } + + // If all returnOptions entries were invalid (e.g. objects lacking a string + // `partial` field), returnItems would be empty. Emitting "RETURN ()" is + // technically valid per RFC 4731 but returns nothing useful. Fall through + // to the legacy SEARCH path instead so the caller gets a usable result. + if (returnItems.length > 0) { + const returnClause = [ + { type: 'ATOM', value: 'RETURN' }, + returnItems + ]; + + let esearchResult = {}; + let response; + try { + response = await connection.exec( + options.uid ? 'UID SEARCH' : 'SEARCH', + [...returnClause, ...attributes], + { + untagged: { + ESEARCH: async untagged => { + if (!untagged || !untagged.attributes) return; + // Strip leading (TAG "X") list and optional UID atom. + // The IMAP parser represents parenthesized groups as + // plain Arrays, not objects with type: 'LIST'. + let attrs = untagged.attributes; + let start = 0; + if (attrs[start] && (Array.isArray(attrs[start]) || attrs[start].type === 'LIST')) start++; + if (attrs[start] && typeof attrs[start].value === 'string' && attrs[start].value.toUpperCase() === 'UID') start++; + esearchResult = parseEsearchResponse(attrs.slice(start)); + } + } + } + ); + response.next(); + return esearchResult; + } catch (err) { + await enhanceCommandError(err); + connection.log.warn({ err, cid: connection.id }); + return false; + } + } + // returnItems was empty — fall through to legacy SEARCH path below + } + + // ── Legacy SEARCH path (no returnOptions, or server lacks ESEARCH) ──── // Use a Set to deduplicate sequence numbers/UIDs -- servers may return // duplicates across multiple untagged SEARCH responses. let results = new Set(); @@ -63,3 +190,6 @@ module.exports = async (connection, query, options) => { return false; } }; + +// Exported for unit testing — not intended as public library API +module.exports.parseEsearchResponse = parseEsearchResponse; diff --git a/lib/imap-flow.d.ts b/lib/imap-flow.d.ts index 29001d1..28c8187 100644 --- a/lib/imap-flow.d.ts +++ b/lib/imap-flow.d.ts @@ -606,6 +606,25 @@ export interface ResponseEvent { code?: string; } +/** Result object returned by ESEARCH (RFC 4731) when returnOptions is specified */ +export interface ESearchResult { + /** Total number of matching messages */ + count?: number; + /** Lowest matching UID */ + min?: number; + /** Highest matching UID */ + max?: number; + /** All matching UIDs as compact sequence-set string (e.g. "1,5:10,20") */ + all?: string; + /** Paged subset (RFC 9394 PARTIAL) */ + partial?: { + /** The requested range, e.g. "1:100" */ + range: string; + /** Matching UIDs in that range as compact sequence-set */ + messages: string; + }; +} + export class AuthenticationFailure extends Error { authenticationFailed: true; serverResponseCode?: string; @@ -725,8 +744,13 @@ export class ImapFlow extends EventEmitter { /** Moves messages from current mailbox to destination mailbox */ messageMove(range: SequenceString | number[] | SearchObject, destination: string, options?: { uid?: boolean }): Promise; - /** Search messages from the currently opened mailbox */ + /** Search messages from the currently opened mailbox — returns number[] (backward-compatible) */ search(query: SearchObject, options?: { uid?: boolean }): Promise; + /** Search messages with ESEARCH RETURN options — returns ESearchResult */ + search(query: SearchObject, options: { + uid?: boolean; + returnOptions: Array<'MIN' | 'MAX' | 'COUNT' | 'ALL' | { partial: string }>; + }): Promise; /** Fetch messages from the currently opened mailbox */ fetch(range: SequenceString | number[] | SearchObject, query: FetchQueryObject, options?: FetchOptions): AsyncIterableIterator; diff --git a/lib/imap-flow.js b/lib/imap-flow.js index 6ab1d7d..a25ae8f 100644 --- a/lib/imap-flow.js +++ b/lib/imap-flow.js @@ -2599,7 +2599,42 @@ class ImapFlow extends EventEmitter { return; } - return (await this.run('SEARCH', query, options)) || false; + const result = (await this.run('SEARCH', query, options)) || false; + + // When returnOptions was requested but server lacked ESEARCH capability, + // search.js returns a plain number[]. Derive ESearchResult client-side. + if (options && options.returnOptions && Array.isArray(result)) { + const arr = result; + // Normalize to uppercase so callers can use mixed-case strings like 'count' + const normalizedOptions = options.returnOptions.map(o => + typeof o === 'string' ? o.toUpperCase() : o + ); + const esearch = {}; + if (normalizedOptions.includes('COUNT')) { + esearch.count = arr.length; + } + if (normalizedOptions.includes('MIN') && arr.length) { + esearch.min = arr[0]; // already sorted ascending by search.js + } + if (normalizedOptions.includes('MAX') && arr.length) { + esearch.max = arr[arr.length - 1]; + } + if (normalizedOptions.includes('ALL') && arr.length) { + esearch.all = packMessageRange(arr); + } + // PARTIAL cannot be derived client-side — omit it. + // When returnOptions contains only { partial: ... } items and the server + // lacks ESEARCH, PARTIAL cannot be derived client-side. Return the raw + // number[] so the caller has actionable data. Note: this is an edge case + // — callers targeting no-ESEARCH servers should avoid requesting PARTIAL + // without COUNT or ALL. + if (Object.keys(esearch).length === 0) { + return result; + } + return esearch; + } + + return result; } /** diff --git a/test/search-test.js b/test/search-test.js new file mode 100644 index 0000000..cdf8974 --- /dev/null +++ b/test/search-test.js @@ -0,0 +1,228 @@ +'use strict'; + +const searchCmd = require('../lib/commands/search'); +const { parseEsearchResponse } = searchCmd; +const { ImapFlow } = require('../lib/imap-flow'); + +// Mock connection — capabilities is Map (matches real ImapFlow) +function makeConnection({ hasEsearch = true } = {}) { + const caps = new Map(); + if (hasEsearch) { + caps.set('ESEARCH', true); + } + return { + state: 'SELECTED', + states: { SELECTED: 'SELECTED' }, + capabilities: caps, + exec: async () => ({ next: () => {} }), + log: { warn: () => {} } + }; +} + +// ── Parser tests ─────────────────────────────────────────────────────────── + +module.exports['ESEARCH: parseEsearchResponse COUNT only'] = test => { + const attrs = [ + { type: 'ATOM', value: 'COUNT' }, + { type: 'ATOM', value: '42' } + ]; + const result = parseEsearchResponse(attrs); + test.equal(result.count, 42); + test.equal(result.min, undefined); + test.equal(result.max, undefined); + test.done(); +}; + +module.exports['ESEARCH: parseEsearchResponse MIN MAX'] = test => { + const attrs = [ + { type: 'ATOM', value: 'MIN' }, + { type: 'ATOM', value: '1001' }, + { type: 'ATOM', value: 'MAX' }, + { type: 'ATOM', value: '9876' } + ]; + const result = parseEsearchResponse(attrs); + test.equal(result.min, 1001); + test.equal(result.max, 9876); + test.done(); +}; + +module.exports['ESEARCH: parseEsearchResponse ALL keeps compact string'] = test => { + const attrs = [ + { type: 'ATOM', value: 'ALL' }, + { type: 'ATOM', value: '1001,1005:1010,1020' } + ]; + const result = parseEsearchResponse(attrs); + // Must be preserved as compact string — NOT an array + test.equal(typeof result.all, 'string'); + test.equal(result.all, '1001,1005:1010,1020'); + test.done(); +}; + +module.exports['ESEARCH: parseEsearchResponse PARTIAL'] = test => { + // PARTIAL value arrives as a nested list: (rangeAtom seqSetAtom) + const attrs = [ + { type: 'ATOM', value: 'PARTIAL' }, + { + type: 'LIST', + attributes: [ + { type: 'ATOM', value: '1:100' }, + { type: 'ATOM', value: '1001,1003:1010,1015' } + ] + } + ]; + const result = parseEsearchResponse(attrs); + test.deepEqual(result.partial, { range: '1:100', messages: '1001,1003:1010,1015' }); + test.done(); +}; + +module.exports['ESEARCH: parseEsearchResponse COUNT + PARTIAL combined'] = test => { + const attrs = [ + { type: 'ATOM', value: 'COUNT' }, + { type: 'ATOM', value: '34201' }, + { type: 'ATOM', value: 'PARTIAL' }, + { + type: 'LIST', + attributes: [ + { type: 'ATOM', value: '1:100' }, + { type: 'ATOM', value: '2001,2003:2020' } + ] + } + ]; + const result = parseEsearchResponse(attrs); + test.equal(result.count, 34201); + test.deepEqual(result.partial, { range: '1:100', messages: '2001,2003:2020' }); + test.done(); +}; + +// ── Command-building tests ───────────────────────────────────────────────── + +module.exports['ESEARCH: emits RETURN clause when returnOptions present and server has ESEARCH'] = test => { + const conn = makeConnection({ hasEsearch: true }); + let capturedCommand = null; + let capturedAttributes = null; + conn.exec = async (command, attributes) => { + capturedCommand = command; + capturedAttributes = JSON.stringify(attributes); + return { next: () => {} }; + }; + searchCmd(conn, { seen: false }, { uid: true, returnOptions: ['COUNT'] }).then(() => { + test.equal(capturedCommand, 'UID SEARCH'); + test.ok(capturedAttributes.includes('"RETURN"'), 'should include RETURN atom'); + test.ok(capturedAttributes.includes('"COUNT"'), 'should include COUNT in return list'); + test.done(); + }); +}; + +module.exports['ESEARCH: RETURN clause includes PARTIAL range atom'] = test => { + const conn = makeConnection({ hasEsearch: true }); + let capturedAttributes = null; + conn.exec = async (command, attributes) => { + capturedAttributes = JSON.stringify(attributes); + return { next: () => {} }; + }; + searchCmd(conn, { seen: false }, { uid: true, returnOptions: [{ partial: '1:100' }] }).then(() => { + test.ok(capturedAttributes.includes('"PARTIAL"'), 'should include PARTIAL atom'); + test.ok(capturedAttributes.includes('"1:100"'), 'should include range string'); + test.done(); + }); +}; + +module.exports['ESEARCH: no RETURN clause when server lacks ESEARCH capability'] = test => { + const conn = makeConnection({ hasEsearch: false }); + let capturedAttributes = null; + conn.exec = async (command, attributes, handlers) => { + capturedAttributes = JSON.stringify(attributes); + // Simulate plain SEARCH response + const searchHandler = handlers && handlers.untagged && handlers.untagged.SEARCH; + if (searchHandler) { + await searchHandler({ + attributes: [{ value: '1' }, { value: '2' }, { value: '3' }] + }); + } + return { next: () => {} }; + }; + searchCmd(conn, { seen: false }, { uid: true, returnOptions: ['COUNT', 'ALL'] }).then(result => { + test.ok(!capturedAttributes.includes('"RETURN"'), 'should NOT include RETURN when no ESEARCH'); + test.ok(Array.isArray(result), 'should return number[] when ESEARCH unavailable'); + test.deepEqual(result, [1, 2, 3]); + test.done(); + }); +}; + +module.exports['ESEARCH: parseEsearchResponse ignores unknown keywords'] = test => { + // Dovecot with CONDSTORE may append MODSEQ to ESEARCH responses + const attrs = [ + { type: 'ATOM', value: 'COUNT' }, + { type: 'ATOM', value: '5' }, + { type: 'ATOM', value: 'MODSEQ' }, + { type: 'ATOM', value: '12345' } + ]; + const result = parseEsearchResponse(attrs); + test.equal(result.count, 5); + test.equal(result.modseq, undefined, 'unknown keys should not appear in result'); + test.done(); +}; + +module.exports['ESEARCH: backward compat — no returnOptions returns number[]'] = test => { + const conn = makeConnection({ hasEsearch: true }); + conn.exec = async (command, attributes, handlers) => { + const searchHandler = handlers && handlers.untagged && handlers.untagged.SEARCH; + if (searchHandler) { + await searchHandler({ + attributes: [{ value: '10' }, { value: '20' }] + }); + } + return { next: () => {} }; + }; + // No returnOptions — must return number[] even if server has ESEARCH + searchCmd(conn, { seen: true }, { uid: true }).then(result => { + test.ok(Array.isArray(result)); + test.deepEqual(result, [10, 20]); + test.done(); + }); +}; + +// ── imap-flow.js public API fallback test ───────────────────────────────── +module.exports['imap-flow: search() derives ESearchResult when server has no ESEARCH'] = test => { + const client = new ImapFlow({ + host: 'imap.example.com', + port: 993, + auth: { user: 'test', pass: 'test' }, + logger: false + }); + + // Simulate a selected mailbox and no ESEARCH capability + client.mailbox = { path: 'INBOX' }; + client.state = client.states.SELECTED; + client.capabilities = new Map(); // no ESEARCH + + // Stub run() to return a sorted number[] + client.run = async () => [10, 20, 30, 40, 50]; + + client.search({ seen: false }, { uid: true, returnOptions: ['COUNT', 'MIN', 'MAX', 'ALL'] }).then(result => { + test.equal(typeof result, 'object', 'should return object, not array'); + test.ok(!Array.isArray(result), 'should not be an array'); + test.equal(result.count, 5); + test.equal(result.min, 10); + test.equal(result.max, 50); + // packMessageRange([10,20,30,40,50]) → "10,20,30,40,50" (non-contiguous) + test.ok(typeof result.all === 'string' && result.all.length > 0, 'all should be non-empty compact string'); + test.done(); + }); +}; + +module.exports['imap-flow: search() fallback with empty result set'] = test => { + const client = new ImapFlow({ + host: 'imap.example.com', port: 993, + auth: { user: 'test', pass: 'test' }, logger: false + }); + client.mailbox = { path: 'INBOX' }; + client.state = client.states.SELECTED; + client.capabilities = new Map(); + client.run = async () => []; + client.search({}, { uid: true, returnOptions: ['COUNT', 'ALL'] }).then(result => { + test.equal(result.count, 0); + test.equal(result.all, undefined, 'all should be absent for empty result'); + test.done(); + }); +};