Skip to content

feat: store envelopes as list of threads #11529

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: store envelopes as list of threads #11529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3dc5cfa
feat: store envelopes as list of threads
hamza221 Aug 19, 2025
0100ddb
fixup! feat: store envelopes as list of threads
hamza221 Aug 28, 2025
6e22993
fixup! feat: store envelopes as list of threads
hamza221 Sep 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/Contracts/IMailManager.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public function getImapMessage(Horde_Imap_Client_Socket $client,
*
* @return Message[]
*/
public function getThread(Account $account, string $threadRootId): array;
public function getThread(Account $account, string $threadRootId, string $sortOrder = IMailSearch::ORDER_NEWEST_FIRST): array;

/**
* @param Account $sourceAccount
Expand Down
4 changes: 2 additions & 2 deletions lib/Contracts/IMailSearch.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public function findMessage(Account $account,
* @param string|null $userId
* @param string|null $view
*
* @return Message[]
* @return Message[][]
*
* @throws ClientException
* @throws ServiceException
Expand All @@ -59,7 +59,7 @@ public function findMessages(Account $account,
/**
* Run a search through all mailboxes of a user.
*
* @return Message[]
* @return Message[][]
*
* @throws ClientException
* @throws ServiceException
Expand Down
139 changes: 135 additions & 4 deletions lib/Db/MessageMapper.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -756,10 +756,11 @@
/**
* @param Account $account
* @param string $threadRootId
* @param string $sortOrder
*
* @return Message[]
*/
public function findThread(Account $account, string $threadRootId): array {
public function findThread(Account $account, string $threadRootId, string $sortOrder): array {
$qb = $this->db->getQueryBuilder();
$qb->select('messages.*')
->from($this->getTableName(), 'messages')
Expand All @@ -768,7 +769,7 @@
$qb->expr()->eq('mailboxes.account_id', $qb->createNamedParameter($account->getId(), IQueryBuilder::PARAM_INT)),
$qb->expr()->eq('messages.thread_root_id', $qb->createNamedParameter($threadRootId, IQueryBuilder::PARAM_STR), IQueryBuilder::PARAM_STR)
)
->orderBy('messages.sent_at', 'desc');
->orderBy('messages.sent_at', $sortOrder);

return $this->findRelatedData($this->findEntities($qb), $account->getUserId());
}
Expand Down Expand Up @@ -1273,10 +1274,11 @@
* @param Mailbox $mailbox
* @param string $userId
* @param int[] $ids
* @param string $sortOrder
*
* @return Message[]
*/
public function findByMailboxAndIds(Mailbox $mailbox, string $userId, array $ids): array {
public function findByMailboxAndIds(Mailbox $mailbox, string $userId, array $ids, string $sortOrder): array {
if ($ids === []) {
return [];
}
Expand All @@ -1288,7 +1290,7 @@
$qb->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId()), IQueryBuilder::PARAM_INT),
$qb->expr()->in('id', $qb->createParameter('ids'))
)
->orderBy('sent_at', 'desc');
->orderBy('sent_at', $sortOrder);

$results = [];
foreach (array_chunk($ids, 1000) as $chunk) {
Expand All @@ -1298,6 +1300,71 @@
return array_merge([], ...$results);
}

/**
* @param Account $account
* @param Mailbox $mailbox
* @param string $userId
* @param int[] $ids
* @param string $sortOrder
* @param bool $threadingEnabled
*
* @return Message[][]
*/
public function findMessageListsByMailboxAndIds(Account $account, Mailbox $mailbox, string $userId, array $ids, string $sortOrder, bool $threadingEnabled = false): array {
if ($ids === []) {
return [];
}

$base = $this->findByMailboxAndIds($mailbox, $userId, $ids, $sortOrder);
if ($threadingEnabled === false) {
return array_map(static fn (Message $m) => [$m], $base);
}

$threadRoots = array_unique(
array_map(static fn (Message $m) => $m->getThreadRootId(), $base)
);

$allThreadMsgs = [];
$qb = $this->db->getQueryBuilder();
$qb->select('*')
->from($this->getTableName())
->where(
$qb->expr()->eq('mailbox_id', $qb->createNamedParameter($mailbox->getId(), IQueryBuilder::PARAM_INT)),
$qb->expr()->in('thread_root_id', $qb->createParameter('roots')),
$qb->expr()->notIn('id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))
)
->orderBy('sent_at', $sortOrder);
foreach (array_chunk($threadRoots, 1000) as $chunk) {
$qb->setParameter('roots', $chunk, IQueryBuilder::PARAM_STR_ARRAY);
$allThreadMsgs[] = $this->findEntities($qb);
}
$allThreadMsgs = array_merge($base, ...$allThreadMsgs);

$enriched = $this->findRelatedData(array_values($allThreadMsgs), $userId);

$groups = [];
foreach ($enriched as $m) {
$root = $m->getThreadRootId();
$groups[$root][] = $m;

Check failure on line 1348 in lib/Db/MessageMapper.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

PossiblyNullArrayOffset 

lib/Db/MessageMapper.php:1348:4: PossiblyNullArrayOffset: Cannot access value on variable $groups[$root] using possibly null offset null|string (see https://psalm.dev/125)
}

$orderKeys = [];
foreach ($base as $m) {
$key = $m->getThreadRootId();
if (!isset($orderKeys[$key])) {
$orderKeys[$key] = true;

Check failure on line 1355 in lib/Db/MessageMapper.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

PossiblyNullArrayOffset 

lib/Db/MessageMapper.php:1355:5: PossiblyNullArrayOffset: Cannot access value on variable $orderKeys[$key] using possibly null offset null|string (see https://psalm.dev/125)
}
}

$out = [];
foreach (array_keys($orderKeys) as $k) {
if (isset($groups[$k])) {
$out[] = $groups[$k];
}
}
return $out;
}

/**
* @param string $userId
* @param int[] $ids
Expand Down Expand Up @@ -1325,6 +1392,70 @@
return array_merge([], ...$results);
}


/**
* @param Account $account
* @param string $userId
* @param int[] $ids
* @param string $sortOrder
*
* @return Message[][]
*/
public function findMessageListsByIds(Account $account, string $userId, array $ids, string $sortOrder, bool $threadingEnabled = false): array {
if ($ids === []) {
return [];
}

$base = $this->findByIds($userId, $ids, $sortOrder);

if ($threadingEnabled === false) {
return array_map(static fn (Message $m) => [$m], $base);
}

$threadRoots = array_unique(
array_map(static fn (Message $m) => $m->getThreadRootId(), $base)
);

$allThreadMsgs = [];
$qb = $this->db->getQueryBuilder();
$qb->select('m.*')
->from($this->getTableName(), 'm')
->join('m', 'mail_mailboxes', 'mb', $qb->expr()->eq('m.mailbox_id', 'mb.id', IQueryBuilder::PARAM_INT))
->where(
$qb->expr()->eq('mb.account_id', $qb->createNamedParameter($account->getId(), IQueryBuilder::PARAM_INT)),
$qb->expr()->in('m.thread_root_id', $qb->createParameter('roots')),
$qb->expr()->notIn('m.id', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))
)
->orderBy('m.sent_at', $sortOrder);
foreach (array_chunk($threadRoots, 1000) as $chunk) {
$qb->setParameter('roots', $chunk, IQueryBuilder::PARAM_STR_ARRAY);
$allThreadMsgs[] = $this->findEntities($qb);
}
$allThreadMsgs = array_merge($base, ...$allThreadMsgs);

$enriched = $this->findRelatedData(array_values($allThreadMsgs), $userId);
$groups = [];
foreach ($enriched as $m) {
$root = $m->getThreadRootId();
$groups[$root][] = $m;

Check failure on line 1440 in lib/Db/MessageMapper.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

PossiblyNullArrayOffset 

lib/Db/MessageMapper.php:1440:4: PossiblyNullArrayOffset: Cannot access value on variable $groups[$root] using possibly null offset null|string (see https://psalm.dev/125)
}
$orderKeys = [];
foreach ($base as $m) {
$key = $m->getThreadRootId();
if (!isset($orderKeys[$key])) {
$orderKeys[$key] = true;

Check failure on line 1446 in lib/Db/MessageMapper.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

PossiblyNullArrayOffset 

lib/Db/MessageMapper.php:1446:5: PossiblyNullArrayOffset: Cannot access value on variable $orderKeys[$key] using possibly null offset null|string (see https://psalm.dev/125)
}
}

$out = [];
foreach (array_keys($orderKeys) as $k) {
if (isset($groups[$k])) {
$out[] = $groups[$k];
}
}
return $out;
}

/**
* @param Message[] $messages
*
Expand Down
4 changes: 2 additions & 2 deletions lib/IMAP/PreviewEnhancer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,12 @@
}

/**
* @param Message[] $messages
* @param Message[][] $messages
*
* @return Message[]
* @return Message[][]

Check failure on line 57 in lib/IMAP/PreviewEnhancer.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

InvalidReturnType 

lib/IMAP/PreviewEnhancer.php:57:13: InvalidReturnType: The declared return type 'array<array-key, array<array-key, OCA\Mail\Db\Message>>' for OCA\Mail\IMAP\PreviewEnhancer::process is incorrect, got 'array<array-key, OCA\Mail\Db\Message|array<array-key, OCA\Mail\Db\Message>>' (see https://psalm.dev/011)
*/
public function process(Account $account, Mailbox $mailbox, array $messages, bool $preLoadAvatars = false, ?string $userId = null): array {
$needAnalyze = array_reduce($messages, static function (array $carry, Message $message) {

Check failure on line 60 in lib/IMAP/PreviewEnhancer.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

InvalidArgument 

lib/IMAP/PreviewEnhancer.php:60:73: InvalidArgument: The second param of the closure passed to array_reduce must take array<array-key, OCA\Mail\Db\Message> but only accepts OCA\Mail\Db\Message (see https://psalm.dev/004)
if ($message->getStructureAnalyzed()) {
// Nothing to do
return $carry;
Expand All @@ -68,8 +68,8 @@

if ($preLoadAvatars) {
foreach ($messages as $message) {
$from = $message->getFrom()->first();

Check failure on line 71 in lib/IMAP/PreviewEnhancer.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

InvalidMethodCall 

lib/IMAP/PreviewEnhancer.php:71:23: InvalidMethodCall: Cannot call method on array<array-key, OCA\Mail\Db\Message> variable $message (see https://psalm.dev/091)
if ($message->getAvatar() === null && $from !== null && $from->getEmail() !== null && $userId !== null) {

Check failure on line 72 in lib/IMAP/PreviewEnhancer.php

View workflow job for this annotation

GitHub Actions / static-psalm-analysis dev-master

InvalidMethodCall 

lib/IMAP/PreviewEnhancer.php:72:19: InvalidMethodCall: Cannot call method on array<array-key, OCA\Mail\Db\Message> variable $message (see https://psalm.dev/091)
$avatar = $this->avatarService->getCachedAvatar($from->getEmail(), $userId);
if ($avatar === null) {
$message->setFetchAvatarFromClient(true);
Expand Down
5 changes: 3 additions & 2 deletions lib/Service/MailManager.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
use OCA\Mail\Account;
use OCA\Mail\Attachment;
use OCA\Mail\Contracts\IMailManager;
use OCA\Mail\Contracts\IMailSearch;
use OCA\Mail\Db\Mailbox;
use OCA\Mail\Db\MailboxMapper;
use OCA\Mail\Db\Message;
Expand Down Expand Up @@ -236,8 +237,8 @@ public function getImapMessagesForScheduleProcessing(Account $account,
}

#[\Override]
public function getThread(Account $account, string $threadRootId): array {
return $this->dbMessageMapper->findThread($account, $threadRootId);
public function getThread(Account $account, string $threadRootId, string $sortOrder = IMailSearch::ORDER_NEWEST_FIRST): array {
return $this->dbMessageMapper->findThread($account, $threadRootId, $sortOrder);
}

#[\Override]
Expand Down
31 changes: 19 additions & 12 deletions lib/Service/Search/MailSearch.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public function findMessage(Account $account,
* @param int|null $limit
* @param string|null $view
*
* @return Message[]
* @return Message[][]
*
* @throws ClientException
* @throws ServiceException
Expand All @@ -102,8 +102,9 @@ public function findMessages(Account $account,
if ($cursor !== null) {
$query->setCursor($cursor);
}
$threadingEnabled = $view === self::VIEW_THREADED;
if ($view !== null) {
$query->setThreaded($view === self::VIEW_THREADED);
$query->setThreaded($threadingEnabled);
}
// In flagged we don't want anything but flagged messages
if ($mailbox->isSpecialUse(Horde_Imap_Client::SPECIALUSE_FLAGGED)) {
Expand All @@ -113,17 +114,23 @@ public function findMessages(Account $account,
if (!$mailbox->isSpecialUse(Horde_Imap_Client::SPECIALUSE_TRASH)) {
$query->addFlag(Flag::not(Flag::DELETED));
}

return $this->previewEnhancer->process(
$account,
$mailbox,
$this->messageMapper->findByIds($account->getUserId(),
$this->getIdsLocally($account, $mailbox, $query, $sortOrder, $limit),
$sortOrder,
),
true,
$userId
$messages = $this->messageMapper->findMessageListsByIds($account, $account->getUserId(),
$this->getIdsLocally($account, $mailbox, $query, $sortOrder, $limit),
$sortOrder,
$threadingEnabled
);
$processedMessages = [];
foreach ($messages as $messageList) {
$processedMessages[] = $this->previewEnhancer->process(
$account,
$mailbox,
$messageList,
true,
$userId
);
}

return $processedMessages;
}

/**
Expand Down
Loading
Loading