diff --git a/.eslintrc b/.eslintrc index abd292af1b..6e3cfed9bf 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,3 +1,18 @@ { - "extends": "nodebb" + "parser": "@typescript-eslint/parser", + "extends": [ + "nodebb", + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "plugins": [ + "@typescript-eslint" + ], + "settings": { + "import/resolver": { + "typescript": { + "alwaysTryTypes": true + } + } + } } diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000..f429bbf29d Binary files /dev/null and b/dump.rdb differ diff --git a/src/dummy.js b/src/dummy.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/interfaces/post.js b/src/interfaces/post.js new file mode 100644 index 0000000000..c8ad2e549b --- /dev/null +++ b/src/interfaces/post.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/src/interfaces/post.ts b/src/interfaces/post.ts new file mode 100644 index 0000000000..45958028d4 --- /dev/null +++ b/src/interfaces/post.ts @@ -0,0 +1,28 @@ +export interface IPost { + pid: number, + uid: number, + tid: number, + timestamp: number, + deleted: boolean, + upvotes: number, + downvotes: number, + category: Record, + topic: ITopic, + user: { + username?: string, + }, +} + +export interface ITopic { + postcount?: string, + deleted?: boolean, + category?: Record, + tags?: ITag[] | string[], + cid?: number +} + +export interface ITag { + value?: string +} + + diff --git a/src/interfaces/search.js b/src/interfaces/search.js new file mode 100644 index 0000000000..c8ad2e549b --- /dev/null +++ b/src/interfaces/search.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/src/interfaces/search.ts b/src/interfaces/search.ts new file mode 100644 index 0000000000..f9f66590b2 --- /dev/null +++ b/src/interfaces/search.ts @@ -0,0 +1,24 @@ +export interface ISearchData { + query: string, + searchIn: string, + uid?: number; + hasTags?: string, + categories?: string[], + searchChildren?: boolean, + sortBy?: string, + sortDirection?: string, + matchWords?: string, + returnIds?: number[], + itemsPerPage?: number, + page?: number, + replies?: string, + timeRange?: string, + repliesFilter?: string, + timeFilter?: string, + postedBy?: string +} + +export interface ISearch { + search(data: ISearchData): Promise<{ time: string, [key: string]: unknown }>; +} + diff --git a/src/search.js b/src/search.js index df249ec1f6..b0de4fed57 100644 --- a/src/search.js +++ b/src/search.js @@ -1,357 +1,360 @@ -'use strict'; - -const _ = require('lodash'); - -const db = require('./database'); -const batch = require('./batch'); -const posts = require('./posts'); -const topics = require('./topics'); -const categories = require('./categories'); -const user = require('./user'); -const plugins = require('./plugins'); -const privileges = require('./privileges'); -const utils = require('./utils'); - -const search = module.exports; - -search.search = async function (data) { - const start = process.hrtime(); - data.sortBy = data.sortBy || 'relevance'; - - let result; - if (['posts', 'titles', 'titlesposts', 'bookmarks'].includes(data.searchIn)) { - result = await searchInContent(data); - } else if (data.searchIn === 'users') { - result = await user.search(data); - } else if (data.searchIn === 'categories') { - result = await categories.search(data); - } else if (data.searchIn === 'tags') { - result = await topics.searchAndLoadTags(data); - } else if (data.searchIn) { - result = await plugins.hooks.fire('filter:search.searchIn', { - data, - }); - } else { - throw new Error('[[error:unknown-search-filter]]'); - } - - result.time = (process.elapsedTimeSince(start) / 1000).toFixed(2); - return result; +"use strict"; +/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); }; - -async function searchInContent(data) { - data.uid = data.uid || 0; - - const [searchCids, searchUids] = await Promise.all([ - getSearchCids(data), - getSearchUids(data), - ]); - - async function doSearch(type, searchIn) { - if (searchIn.includes(data.searchIn)) { - const result = await plugins.hooks.fire('filter:search.query', { - index: type, - content: data.query, - matchWords: data.matchWords || 'all', - cid: searchCids, - uid: searchUids, - searchData: data, - ids: [], - }); - return Array.isArray(result) ? result : result.ids; - } - return []; - } - let pids = []; - let tids = []; - const inTopic = String(data.query || '').match(/^in:topic-([\d]+) /); - if (inTopic) { - const tid = inTopic[1]; - const cleanedTerm = data.query.replace(inTopic[0], ''); - pids = await topics.search(tid, cleanedTerm); - } else if (data.searchIn === 'bookmarks') { - pids = await searchInBookmarks(data, searchCids, searchUids); - } else { - [pids, tids] = await Promise.all([ - doSearch('post', ['posts', 'titlesposts']), - doSearch('topic', ['titles', 'titlesposts']), - ]); - } - - const mainPids = await topics.getMainPids(tids); - - let allPids = mainPids.concat(pids).filter(Boolean); - - allPids = await privileges.posts.filter('topics:read', allPids, data.uid); - allPids = await filterAndSort(allPids, data); - - const metadata = await plugins.hooks.fire('filter:search.inContent', { - pids: allPids, - data: data, - }); - - if (data.returnIds) { - const mainPidsSet = new Set(mainPids); - const mainPidToTid = _.zipObject(mainPids, tids); - const pidsSet = new Set(pids); - const returnPids = allPids.filter(pid => pidsSet.has(pid)); - const returnTids = allPids.filter(pid => mainPidsSet.has(pid)).map(pid => mainPidToTid[pid]); - return { pids: returnPids, tids: returnTids }; - } - - const itemsPerPage = Math.min(data.itemsPerPage || 10, 100); - const returnData = { - posts: [], - matchCount: metadata.pids.length, - pageCount: Math.max(1, Math.ceil(parseInt(metadata.pids.length, 10) / itemsPerPage)), - }; - - if (data.page) { - const start = Math.max(0, (data.page - 1)) * itemsPerPage; - metadata.pids = metadata.pids.slice(start, start + itemsPerPage); - } - - returnData.posts = await posts.getPostSummaryByPids(metadata.pids, data.uid, {}); - await plugins.hooks.fire('filter:search.contentGetResult', { result: returnData, data: data }); - delete metadata.pids; - delete metadata.data; - return Object.assign(returnData, metadata); -} - -async function searchInBookmarks(data, searchCids, searchUids) { - const { uid, query, matchWords } = data; - const allPids = []; - await batch.processSortedSet(`uid:${uid}:bookmarks`, async (pids) => { - if (Array.isArray(searchCids) && searchCids.length) { - pids = await posts.filterPidsByCid(pids, searchCids); - } - if (Array.isArray(searchUids) && searchUids.length) { - pids = await posts.filterPidsByUid(pids, searchUids); - } - if (query) { - const tokens = String(query).split(' '); - const postData = await db.getObjectsFields(pids.map(pid => `post:${pid}`), ['content', 'tid']); - const tids = _.uniq(postData.map(p => p.tid)); - const topicData = await db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['title']); - const tidToTopic = _.zipObject(tids, topicData); - pids = pids.filter((pid, i) => { - const content = String(postData[i].content); - const title = String(tidToTopic[postData[i].tid].title); - const method = (matchWords === 'any' ? 'some' : 'every'); - return tokens[method]( - token => content.includes(token) || title.includes(token) - ); - }); - } - allPids.push(...pids); - }, { - batch: 500, - }); - - return allPids; -} - -async function filterAndSort(pids, data) { - if (data.sortBy === 'relevance' && - !data.replies && - !data.timeRange && - !data.hasTags && - data.searchIn !== 'bookmarks' && - !plugins.hooks.hasListeners('filter:search.filterAndSort')) { - return pids; - } - let postsData = await getMatchedPosts(pids, data); - if (!postsData.length) { - return pids; - } - postsData = postsData.filter(Boolean); - - postsData = filterByPostcount(postsData, data.replies, data.repliesFilter); - postsData = filterByTimerange(postsData, data.timeRange, data.timeFilter); - postsData = filterByTags(postsData, data.hasTags); - - sortPosts(postsData, data); - - const result = await plugins.hooks.fire('filter:search.filterAndSort', { pids: pids, posts: postsData, data: data }); - return result.posts.map(post => post && post.pid); +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +const lodash_1 = require("lodash"); +const batch_1 = __importDefault(require("./batch")); +const categories_1 = __importDefault(require("./categories")); +const database_1 = __importDefault(require("./database")); +const plugins_1 = __importDefault(require("./plugins")); +const posts_1 = __importDefault(require("./posts")); +const privileges_1 = __importDefault(require("./privileges")); +const promisify_1 = __importDefault(require("./promisify")); +const topics_1 = __importDefault(require("./topics")); +const user_1 = __importDefault(require("./user")); +const utils_1 = __importDefault(require("./utils")); +function getWatchedCids(data) { + return __awaiter(this, void 0, void 0, function* () { + if (!data.categories.includes('watched')) { + return []; + } + return yield user_1.default.getWatchedCategories(data.uid); + }); } - -async function getMatchedPosts(pids, data) { - const postFields = ['pid', 'uid', 'tid', 'timestamp', 'deleted', 'upvotes', 'downvotes']; - - let postsData = await posts.getPostsFields(pids, postFields); - postsData = postsData.filter(post => post && !post.deleted); - const uids = _.uniq(postsData.map(post => post.uid)); - const tids = _.uniq(postsData.map(post => post.tid)); - - const [users, topics] = await Promise.all([ - getUsers(uids, data), - getTopics(tids, data), - ]); - - const tidToTopic = _.zipObject(tids, topics); - const uidToUser = _.zipObject(uids, users); - postsData.forEach((post) => { - if (topics && tidToTopic[post.tid]) { - post.topic = tidToTopic[post.tid]; - if (post.topic && post.topic.category) { - post.category = post.topic.category; - } - } - - if (uidToUser[post.uid]) { - post.user = uidToUser[post.uid]; - } - }); - - return postsData.filter(post => post && post.topic && !post.topic.deleted); +function getChildrenCids(data) { + return __awaiter(this, void 0, void 0, function* () { + if (!data.searchChildren) { + return []; + } + const childrenCids = yield Promise.all(data.categories.map(cid => categories_1.default.getChildrenCids(cid))); + return yield privileges_1.default.categories.filterCids('find', (0, lodash_1.uniq)((0, lodash_1.flatten)(childrenCids)), data.uid); + }); } - -async function getUsers(uids, data) { - if (data.sortBy.startsWith('user')) { - return user.getUsersFields(uids, ['username']); - } - return []; +function getSearchUids(data) { + return __awaiter(this, void 0, void 0, function* () { + if (!data.postedBy) { + return []; + } + return yield user_1.default.getUidsByUsernames(Array.isArray(data.postedBy) ? data.postedBy : [data.postedBy]); + }); } - -async function getTopics(tids, data) { - const topicsData = await topics.getTopicsData(tids); - const cids = _.uniq(topicsData.map(topic => topic && topic.cid)); - const categories = await getCategories(cids, data); - - const cidToCategory = _.zipObject(cids, categories); - topicsData.forEach((topic) => { - if (topic && categories && cidToCategory[topic.cid]) { - topic.category = cidToCategory[topic.cid]; - } - if (topic && topic.tags) { - topic.tags = topic.tags.map(tag => tag.value); - } - }); - - return topicsData; +function getSearchCids(data) { + return __awaiter(this, void 0, void 0, function* () { + if (!Array.isArray(data.categories) || !data.categories.length) { + return []; + } + if (data.categories.includes('all')) { + return yield categories_1.default.getCidsByPrivilege('categories:cid', data.uid, 'read'); + } + const [watchedCids, childrenCids] = yield Promise.all([ + getWatchedCids(data), + getChildrenCids(data), + ]); + const concatenatedData = [...watchedCids, ...childrenCids, ...data.categories]; + return (0, lodash_1.uniq)(concatenatedData.filter(Boolean)); + }); } - -async function getCategories(cids, data) { - const categoryFields = []; - - if (data.sortBy.startsWith('category.')) { - categoryFields.push(data.sortBy.split('.')[1]); - } - if (!categoryFields.length) { - return null; - } - - return await db.getObjectsFields(cids.map(cid => `category:${cid}`), categoryFields); +function searchInBookmarks(data, searchCids, searchUids) { + return __awaiter(this, void 0, void 0, function* () { + const { uid, query, matchWords } = data; + const allPids = []; + yield batch_1.default.processSortedSet(`uid:${uid}:bookmarks`, (pids) => __awaiter(this, void 0, void 0, function* () { + if (Array.isArray(searchCids) && searchCids.length) { + pids = yield posts_1.default.filterPidsByCid(pids, searchCids); + } + if (Array.isArray(searchUids) && searchUids.length) { + pids = yield posts_1.default.filterPidsByUid(pids, searchUids); + } + if (query) { + const tokens = query.toString().split(' '); + const postData = yield database_1.default.getObjectsFields(pids.map(pid => `post:${pid}`), ['content', 'tid']); + const tids = (0, lodash_1.uniq)(postData.map((p) => p.tid)); + const topicData = yield database_1.default.getObjectsFields(tids.map(tid => `topic:${tid}`), ['title']); + const tidToTopic = (0, lodash_1.zipObject)(tids, topicData); + pids = pids.filter((_, i) => { + const content = JSON.stringify(postData[i].content); + const title = `${tidToTopic[postData[i].tid].title}`; + const method = (matchWords === 'any' ? 'some' : 'every'); + return tokens[method](token => content.includes(token) || title.includes(token)); + }); + } + allPids.push(...pids); + }), { + batch: 500, + }); + return allPids; + }); } - function filterByPostcount(posts, postCount, repliesFilter) { - postCount = parseInt(postCount, 10); - if (postCount) { - if (repliesFilter === 'atleast') { - posts = posts.filter(post => post.topic && post.topic.postcount >= postCount); - } else { - posts = posts.filter(post => post.topic && post.topic.postcount <= postCount); - } - } - return posts; + const parsedPostCount = parseInt(postCount, 10); + if (postCount) { + const filterCondition = repliesFilter === 'atleast' ? + (post) => Number(post === null || post === void 0 ? void 0 : post.topic.postcount) >= parsedPostCount : + (post) => Number(post === null || post === void 0 ? void 0 : post.topic.postcount) <= parsedPostCount; + posts = posts.filter(filterCondition); + } + return posts; } - function filterByTimerange(posts, timeRange, timeFilter) { - timeRange = parseInt(timeRange, 10) * 1000; - if (timeRange) { - const time = Date.now() - timeRange; - if (timeFilter === 'newer') { - posts = posts.filter(post => post.timestamp >= time); - } else { - posts = posts.filter(post => post.timestamp <= time); - } - } - return posts; + const parsedTimeRange = parseInt(timeRange, 10) * 1000; + if (timeRange) { + const time = Date.now() - parsedTimeRange; + if (timeFilter === 'newer') { + posts = posts.filter(post => post.timestamp >= time); + } + else { + posts = posts.filter(post => post.timestamp <= time); + } + } + return posts; } - function filterByTags(posts, hasTags) { - if (Array.isArray(hasTags) && hasTags.length) { - posts = posts.filter((post) => { - let hasAllTags = false; - if (post && post.topic && Array.isArray(post.topic.tags) && post.topic.tags.length) { - hasAllTags = hasTags.every(tag => post.topic.tags.includes(tag)); - } - return hasAllTags; - }); - } - return posts; + if (Array.isArray(hasTags) && hasTags.length) { + posts = posts.filter((post) => { + let hasAllTags = false; + if (post && post.topic && Array.isArray(post.topic.tags) && post.topic.tags.length) { + hasAllTags = hasTags.every((tag) => post.topic.tags.includes(tag)); + } + return hasAllTags; + }); + } + return posts; } - function sortPosts(posts, data) { - if (!posts.length || data.sortBy === 'relevance') { - return; - } - - data.sortDirection = data.sortDirection || 'desc'; - const direction = data.sortDirection === 'desc' ? 1 : -1; - const fields = data.sortBy.split('.'); - if (fields.length === 1) { - return posts.sort((p1, p2) => direction * (p2[fields[0]] - p1[fields[0]])); - } - - const firstPost = posts[0]; - if (!fields || fields.length !== 2 || !firstPost[fields[0]] || !firstPost[fields[0]][fields[1]]) { - return; - } - - const isNumeric = utils.isNumber(firstPost[fields[0]][fields[1]]); - - if (isNumeric) { - posts.sort((p1, p2) => direction * (p2[fields[0]][fields[1]] - p1[fields[0]][fields[1]])); - } else { - posts.sort((p1, p2) => { - if (p1[fields[0]][fields[1]] > p2[fields[0]][fields[1]]) { - return direction; - } else if (p1[fields[0]][fields[1]] < p2[fields[0]][fields[1]]) { - return -direction; - } - return 0; - }); - } + var _a; + if (!posts.length || data.sortBy === 'relevance') { + return; + } + data.sortDirection = data.sortDirection || 'desc'; + const direction = data.sortDirection === 'desc' ? 1 : -1; + const fields = data.sortBy.split('.'); + if (fields.length === 1) { + return posts.sort((post_1, post_2) => direction * (post_2[fields[0]] - post_1[fields[0]])); + } + const firstPost = posts[0]; + const isValid = fields && fields.length === 2 && ((_a = firstPost === null || firstPost === void 0 ? void 0 : firstPost[fields[0]]) === null || _a === void 0 ? void 0 : _a[fields[1]]); + if (!isValid) { + return; + } + const isNumeric = utils_1.default.isNumber(firstPost[fields[0]][fields[1]]); + if (isNumeric) { + posts.sort((post_1, post_2) => direction * (post_2[fields[0]][fields[1]] - post_1[fields[0]][fields[1]])); + } + else { + posts.sort((post_1, post_2) => { + if (post_1[fields[0]][fields[1]] > post_2[fields[0]][fields[1]]) { + return direction; + } + else if (post_1[fields[0]][fields[1]] < post_2[fields[0]][fields[1]]) { + return -direction; + } + return 0; + }); + } } - -async function getSearchCids(data) { - if (!Array.isArray(data.categories) || !data.categories.length) { - return []; - } - - if (data.categories.includes('all')) { - return await categories.getCidsByPrivilege('categories:cid', data.uid, 'read'); - } - - const [watchedCids, childrenCids] = await Promise.all([ - getWatchedCids(data), - getChildrenCids(data), - ]); - return _.uniq(watchedCids.concat(childrenCids).concat(data.categories).filter(Boolean)); +function getUsers(uids, data) { + return __awaiter(this, void 0, void 0, function* () { + if (data.sortBy.startsWith('user')) { + return yield user_1.default.getUsersFields(uids, ['username']); + } + return []; + }); } - -async function getWatchedCids(data) { - if (!data.categories.includes('watched')) { - return []; - } - return await user.getWatchedCategories(data.uid); +function getCategories(cids, data) { + return __awaiter(this, void 0, void 0, function* () { + const categoryFields = []; + if (data.sortBy.startsWith('category.')) { + categoryFields.push(data.sortBy.split('.')[1]); + } + if (!categoryFields.length) { + return null; + } + return yield database_1.default.getObjectsFields(cids.map(cid => `category:${cid}`), categoryFields); + }); } - -async function getChildrenCids(data) { - if (!data.searchChildren) { - return []; - } - const childrenCids = await Promise.all(data.categories.map(cid => categories.getChildrenCids(cid))); - return await privileges.categories.filterCids('find', _.uniq(_.flatten(childrenCids)), data.uid); +function getTopics(tids, data) { + return __awaiter(this, void 0, void 0, function* () { + const topicsData = yield topics_1.default.getTopicsData(tids); + const cids = (0, lodash_1.uniq)(topicsData.map((topic) => topic && topic.cid)); + const categories = yield getCategories(cids, data); + const cidToCategory = (0, lodash_1.zipObject)(cids, categories); + topicsData.forEach((topic) => { + if (topic && categories && cidToCategory[topic.cid]) { + topic.category = cidToCategory[topic.cid]; + } + if (Array.isArray(topic.tags) && topic.tags.length > 0 && typeof topic.tags[0] !== 'string') { + topic.tags = topic.tags.map((tag) => tag.value); + } + }); + return topicsData; + }); } - -async function getSearchUids(data) { - if (!data.postedBy) { - return []; - } - return await user.getUidsByUsernames(Array.isArray(data.postedBy) ? data.postedBy : [data.postedBy]); +function getMatchedPosts(pids, data) { + return __awaiter(this, void 0, void 0, function* () { + const postFields = ['pid', 'uid', 'tid', 'timestamp', 'deleted', 'upvotes', 'downvotes']; + let postsData = yield posts_1.default.getPostsFields(pids, postFields); + postsData = postsData.filter((post) => post && !post.deleted); + const uids = (0, lodash_1.uniq)(postsData.map((post) => post.uid)); + const tids = (0, lodash_1.uniq)(postsData.map((post) => post.tid)); + const [users, topics] = yield Promise.all([ + getUsers(uids, data), + getTopics(tids, data), + ]); + const tidToTopic = (0, lodash_1.zipObject)(tids, topics); + const uidToUser = (0, lodash_1.zipObject)(uids, users); + postsData.forEach((post) => { + if (topics && tidToTopic[post.tid]) { + post.topic = tidToTopic[post.tid]; + if (post.topic && post.topic.category) { + post.category = post.topic.category; + } + } + if (uidToUser[post.uid]) { + post.user = uidToUser[post.uid]; + } + }); + return postsData.filter((post) => post && post.topic && !post.topic.deleted); + }); } - -require('./promisify')(search); +function filterAndSort(pids, data) { + return __awaiter(this, void 0, void 0, function* () { + if (data.sortBy === 'relevance' && + !data.replies && + !data.timeRange && + !data.hasTags && + data.searchIn !== 'bookmarks' && + !plugins_1.default.hooks.hasListeners('filter:search.filterAndSort')) { + return pids; + } + let postsData = yield getMatchedPosts(pids, data); + if (!postsData.length) { + return pids; + } + postsData = postsData.filter(Boolean); + postsData = filterByPostcount(postsData, data.replies, data.repliesFilter); + postsData = filterByTimerange(postsData, data.timeRange, data.timeFilter); + postsData = filterByTags(postsData, data.hasTags); + sortPosts(postsData, data); + const result = yield plugins_1.default.hooks.fire('filter:search.filterAndSort', { pids: pids, posts: postsData, data: data }); + return result.posts.map((post) => post && post.pid); + }); +} +function searchInContent(data) { + return __awaiter(this, void 0, void 0, function* () { + data.uid = data.uid || 0; + const [searchCids, searchUids] = yield Promise.all([ + getSearchCids(data), + getSearchUids(data), + ]); + function doSearch(type, searchIn) { + return __awaiter(this, void 0, void 0, function* () { + if (searchIn.includes(data.searchIn)) { + const result = yield plugins_1.default.hooks.fire('filter:search.query', { + index: type, + content: data.query, + matchWords: data.matchWords || 'all', + cid: searchCids, + uid: searchUids, + searchData: data, + ids: [], + }); + return Array.isArray(result) ? result : result.ids; + } + return []; + }); + } + let pids = []; + let tids = []; + const inTopic = `${data.query || ''}`.match(/^in:topic-([\d]+) /); + if (inTopic) { + const tid = inTopic[1]; + const cleanedTerm = data.query.replace(inTopic[0], ''); + pids = yield topics_1.default.search(tid, cleanedTerm); + } + else if (data.searchIn === 'bookmarks') { + pids = yield searchInBookmarks(data, searchCids, searchUids); + } + else { + [pids, tids] = yield Promise.all([ + doSearch('post', ['posts', 'titlesposts']), + doSearch('topic', ['titles', 'titlesposts']), + ]); + } + const mainPids = yield topics_1.default.getMainPids(tids); + let allPids = mainPids.concat(pids).filter(Boolean); + allPids = yield privileges_1.default.posts.filter('topics:read', allPids, data.uid); + allPids = yield filterAndSort(allPids, data); + const metadata = yield plugins_1.default.hooks.fire('filter:search.inContent', { + pids: allPids, + data: data, + }); + if (data.returnIds) { + const mainPidsSet = new Set(mainPids); + const mainPidToTid = (0, lodash_1.zipObject)(mainPids, tids); + const pidsSet = new Set(pids); + const returnPids = allPids.filter((pid) => pidsSet.has(pid)); + const returnTids = allPids.filter((pid) => mainPidsSet.has(pid)).map(pid => mainPidToTid[pid]); + return { pids: returnPids, tids: returnTids }; + } + const itemsPerPage = Math.min(data.itemsPerPage || 10, 100); + const returnData = { + posts: [], + matchCount: metadata.pids.length, + pageCount: Math.max(1, Math.ceil(parseInt(metadata.pids.length, 10) / itemsPerPage)), + }; + if (data.page) { + const start = Math.max(0, (data.page - 1)) * itemsPerPage; + metadata.pids = metadata.pids.slice(start, start + itemsPerPage); + } + returnData.posts = yield posts_1.default.getPostSummaryByPids(metadata.pids, data.uid, {}); + yield plugins_1.default.hooks.fire('filter:search.contentGetResult', { result: returnData, data: data }); + delete metadata.pids; + delete metadata.data; + return Object.assign(returnData, metadata); + }); +} +const search = { + search: function (data) { + return __awaiter(this, void 0, void 0, function* () { + const start = process.hrtime(); + data.sortBy = data.sortBy || 'relevance'; + let result; + if (['posts', 'titles', 'titlesposts', 'bookmarks'].includes(data.searchIn)) { + result = yield searchInContent(data); + } + else if (data.searchIn === 'users') { + result = yield user_1.default.search(data); + } + else if (data.searchIn === 'categories') { + result = yield categories_1.default.search(data); + } + else if (data.searchIn === 'tags') { + result = yield topics_1.default.searchAndLoadTags(data); + } + else if (data.searchIn) { + result = yield plugins_1.default.hooks.fire('filter:search.searchIn', { + data, + }); + } + else { + throw new Error('[[error:unknown-search-filter]]'); + } + result.time = (utils_1.default.elapsedTimeSince(start) / 1000).toFixed(2); + return result; + }); + }, +}; +(0, promisify_1.default)(search); +module.exports = search; diff --git a/src/search.ts b/src/search.ts new file mode 100644 index 0000000000..776f394731 --- /dev/null +++ b/src/search.ts @@ -0,0 +1,376 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ + +import { Dictionary, flatten, uniq, zipObject } from 'lodash'; +import batch from './batch'; +import categories from './categories'; +import db from './database'; +import { IPost, ITag, ITopic } from './interfaces/post'; +import { ISearch, ISearchData } from './interfaces/search'; +import plugins from './plugins'; +import posts from './posts'; +import privileges from './privileges'; +import promisify from './promisify'; +import topics from './topics'; +import user from './user'; +import utils from './utils'; + +async function getWatchedCids(data: ISearchData): Promise { + if (!data.categories.includes('watched')) { + return []; + } + return await user.getWatchedCategories(data.uid); +} + +async function getChildrenCids(data: ISearchData): Promise { + if (!data.searchChildren) { + return []; + } + const childrenCids: string[] = await Promise.all(data.categories.map(cid => categories.getChildrenCids(cid))); + return await privileges.categories.filterCids('find', uniq(flatten(childrenCids)), data.uid); +} + +async function getSearchUids(data: ISearchData): Promise { + if (!data.postedBy) { + return []; + } + return await user.getUidsByUsernames(Array.isArray(data.postedBy) ? data.postedBy : [data.postedBy]); +} + +async function getSearchCids(data: ISearchData): Promise<(string | number)[]> { + if (!Array.isArray(data.categories) || !data.categories.length) { + return []; + } + + if (data.categories.includes('all')) { + return await categories.getCidsByPrivilege('categories:cid', data.uid, 'read'); + } + + const [watchedCids, childrenCids] = await Promise.all([ + getWatchedCids(data), + getChildrenCids(data), + ]); + + const concatenatedData = [...watchedCids, ...childrenCids, ...data.categories]; + + return uniq(concatenatedData.filter(Boolean)); +} + +async function searchInBookmarks(data: ISearchData, + searchCids: (string | number)[], searchUids: number[]): Promise { + const { uid, query, matchWords } = data; + const allPids: number[] = []; + await batch.processSortedSet(`uid:${uid}:bookmarks`, async (pids: number[]) => { + if (Array.isArray(searchCids) && searchCids.length) { + pids = await posts.filterPidsByCid(pids, searchCids); + } + + if (Array.isArray(searchUids) && searchUids.length) { + pids = await posts.filterPidsByUid(pids, searchUids); + } + + if (query) { + const tokens = query.toString().split(' '); + const postData: { tid: number; content: unknown }[] = await db.getObjectsFields(pids.map(pid => `post:${pid}`), ['content', 'tid']); + const tids: number[] = uniq(postData.map((p: { tid: number; }) => p.tid)); + const topicData: { title: string }[] = await db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['title']); + const tidToTopic: Dictionary<{ title: string }> = zipObject(tids, topicData); + pids = pids.filter((_, i) => { + const content = JSON.stringify(postData[i].content); + const title = `${tidToTopic[postData[i].tid].title}`; + const method = (matchWords === 'any' ? 'some' : 'every'); + return tokens[method]( + token => content.includes(token) || title.includes(token) + ); + }); + } + allPids.push(...pids); + }, { + batch: 500, + }); + + return allPids; +} + +function filterByPostcount(posts: IPost[], postCount: string, repliesFilter: string): IPost[] { + const parsedPostCount = parseInt(postCount, 10); + if (postCount) { + const filterCondition = repliesFilter === 'atleast' ? + (post: IPost) => Number(post?.topic.postcount) >= parsedPostCount : + (post: IPost) => Number(post?.topic.postcount) <= parsedPostCount; + + posts = posts.filter(filterCondition); + } + return posts; +} + +function filterByTimerange(posts: IPost[], timeRange: string, timeFilter: string): IPost[] { + const parsedTimeRange = parseInt(timeRange, 10) * 1000; + if (timeRange) { + const time = Date.now() - parsedTimeRange; + if (timeFilter === 'newer') { + posts = posts.filter(post => post.timestamp >= time); + } else { + posts = posts.filter(post => post.timestamp <= time); + } + } + return posts; +} + +function filterByTags(posts: IPost[], hasTags: string): IPost[] { + if (Array.isArray(hasTags) && hasTags.length) { + posts = posts.filter((post) => { + let hasAllTags = false; + if (post && post.topic && Array.isArray(post.topic.tags) && post.topic.tags.length) { + hasAllTags = hasTags.every((tag: string) => post.topic.tags.includes(tag)); + } + return hasAllTags; + }); + } + return posts; +} + +function sortPosts(posts: IPost[], data: ISearchData): IPost[] { + if (!posts.length || data.sortBy === 'relevance') { + return; + } + + data.sortDirection = data.sortDirection || 'desc'; + const direction = data.sortDirection === 'desc' ? 1 : -1; + const fields = data.sortBy.split('.'); + + if (fields.length === 1) { + return posts.sort((post_1, post_2) => direction * (post_2[fields[0]] - post_1[fields[0]])); + } + + const firstPost = posts[0]; + const isValid = fields && fields.length === 2 && firstPost?.[fields[0]]?.[fields[1]]; + if (!isValid) { + return; + } + + const isNumeric = utils.isNumber(firstPost[fields[0]][fields[1]]); + + if (isNumeric) { + posts.sort((post_1, post_2) => direction * (post_2[fields[0]][fields[1]] - post_1[fields[0]][fields[1]])); + } else { + posts.sort((post_1, post_2) => { + if (post_1[fields[0]][fields[1]] > post_2[fields[0]][fields[1]]) { + return direction; + } else if (post_1[fields[0]][fields[1]] < post_2[fields[0]][fields[1]]) { + return -direction; + } + return 0; + }); + } +} + +async function getUsers(uids: number[], data: ISearchData): Promise { + if (data.sortBy.startsWith('user')) { + return await user.getUsersFields(uids, ['username']); + } + return []; +} + +async function getCategories(cids: number[], data: ISearchData): Promise[]> { + const categoryFields = []; + + if (data.sortBy.startsWith('category.')) { + categoryFields.push(data.sortBy.split('.')[1]); + } + if (!categoryFields.length) { + return null; + } + + return await db.getObjectsFields(cids.map(cid => `category:${cid}`), categoryFields); +} + +async function getTopics(tids: number[], data: ISearchData): Promise { + const topicsData: ITopic[] = await topics.getTopicsData(tids); + const cids: number[] = uniq(topicsData.map((topic: ITopic) => topic && topic.cid)); + const categories = await getCategories(cids, data); + + const cidToCategory = zipObject(cids, categories); + topicsData.forEach((topic: ITopic) => { + if (topic && categories && cidToCategory[topic.cid]) { + topic.category = cidToCategory[topic.cid]; + } + if (Array.isArray(topic.tags) && topic.tags.length > 0 && typeof topic.tags[0] !== 'string') { + topic.tags = (topic.tags as ITag[]).map((tag: ITag) => tag.value); + } + }); + + return topicsData; +} + +async function getMatchedPosts(pids: number[], data: ISearchData): Promise { + const postFields = ['pid', 'uid', 'tid', 'timestamp', 'deleted', 'upvotes', 'downvotes']; + + let postsData: IPost[] = await posts.getPostsFields(pids, postFields); + postsData = postsData.filter((post: IPost) => post && !post.deleted); + const uids: number[] = uniq(postsData.map((post: IPost) => post.uid)); + const tids: number[] = uniq(postsData.map((post: IPost) => post.tid)); + + const [users, topics] = await Promise.all([ + getUsers(uids, data), + getTopics(tids, data), + ]); + + const tidToTopic = zipObject(tids, topics); + const uidToUser = zipObject(uids, users); + + postsData.forEach((post: IPost) => { + if (topics && tidToTopic[post.tid]) { + post.topic = tidToTopic[post.tid]; + if (post.topic && post.topic.category) { + post.category = post.topic.category; + } + } + + if (uidToUser[post.uid]) { + post.user = uidToUser[post.uid]; + } + }); + + return postsData.filter((post: IPost) => post && post.topic && !post.topic.deleted); +} + +async function filterAndSort(pids: number[], data: ISearchData): Promise { + if (data.sortBy === 'relevance' && + !data.replies && + !data.timeRange && + !data.hasTags && + data.searchIn !== 'bookmarks' && + !plugins.hooks.hasListeners('filter:search.filterAndSort')) { + return pids; + } + let postsData = await getMatchedPosts(pids, data); + if (!postsData.length) { + return pids; + } + postsData = postsData.filter(Boolean); + + postsData = filterByPostcount(postsData, data.replies, data.repliesFilter); + postsData = filterByTimerange(postsData, data.timeRange, data.timeFilter); + postsData = filterByTags(postsData, data.hasTags); + + sortPosts(postsData, data); + + const result = await plugins.hooks.fire('filter:search.filterAndSort', { pids: pids, posts: postsData, data: data }); + return result.posts.map((post: IPost) => post && post.pid); +} + +async function searchInContent(data: ISearchData) { + data.uid = data.uid || 0; + + const [searchCids, searchUids] = await Promise.all([ + getSearchCids(data), + getSearchUids(data), + ]); + + async function doSearch(type: string, searchIn: string[]): Promise { + if (searchIn.includes(data.searchIn)) { + const result = await plugins.hooks.fire('filter:search.query', { + index: type, + content: data.query, + matchWords: data.matchWords || 'all', + cid: searchCids, + uid: searchUids, + searchData: data, + ids: [], + }); + return Array.isArray(result) ? result : result.ids; + } + return []; + } + + let pids = []; + let tids = []; + + const inTopic = `${data.query || ''}`.match(/^in:topic-([\d]+) /); + + if (inTopic) { + const tid = inTopic[1]; + const cleanedTerm = data.query.replace(inTopic[0], ''); + pids = await topics.search(tid, cleanedTerm); + } else if (data.searchIn === 'bookmarks') { + pids = await searchInBookmarks(data, searchCids, searchUids); + } else { + [pids, tids] = await Promise.all([ + doSearch('post', ['posts', 'titlesposts']), + doSearch('topic', ['titles', 'titlesposts']), + ]); + } + + const mainPids = await topics.getMainPids(tids); + + let allPids = mainPids.concat(pids).filter(Boolean); + + allPids = await privileges.posts.filter('topics:read', allPids, data.uid); + allPids = await filterAndSort(allPids, data); + + const metadata = await plugins.hooks.fire('filter:search.inContent', { + pids: allPids, + data: data, + }); + + if (data.returnIds) { + const mainPidsSet = new Set(mainPids); + const mainPidToTid = zipObject(mainPids, tids); + const pidsSet = new Set(pids); + const returnPids = allPids.filter((pid: number) => pidsSet.has(pid)); + const returnTids = allPids.filter((pid: number) => mainPidsSet.has(pid)).map(pid => mainPidToTid[pid]); + return { pids: returnPids, tids: returnTids }; + } + + const itemsPerPage = Math.min(data.itemsPerPage || 10, 100); + + const returnData = { + posts: [], + matchCount: metadata.pids.length, + pageCount: Math.max(1, Math.ceil(parseInt(metadata.pids.length, 10) / itemsPerPage)), + }; + + if (data.page) { + const start = Math.max(0, (data.page - 1)) * itemsPerPage; + metadata.pids = metadata.pids.slice(start, start + itemsPerPage); + } + + returnData.posts = await posts.getPostSummaryByPids(metadata.pids, data.uid, {}); + await plugins.hooks.fire('filter:search.contentGetResult', { result: returnData, data: data }); + delete metadata.pids; + delete metadata.data; + return Object.assign(returnData, metadata); +} + +const search: ISearch = { + search: async function (data: ISearchData) { + const start = process.hrtime(); + data.sortBy = data.sortBy || 'relevance'; + let result: { time: string, [key: string]: unknown }; + + if (['posts', 'titles', 'titlesposts', 'bookmarks'].includes(data.searchIn)) { + result = await searchInContent(data); + } else if (data.searchIn === 'users') { + result = await user.search(data); + } else if (data.searchIn === 'categories') { + result = await categories.search(data); + } else if (data.searchIn === 'tags') { + result = await topics.searchAndLoadTags(data); + } else if (data.searchIn) { + result = await plugins.hooks.fire('filter:search.searchIn', { + data, + }); + } else { + throw new Error('[[error:unknown-search-filter]]'); + } + + result.time = (utils.elapsedTimeSince(start) / 1000).toFixed(2); + return result; + }, +}; + +promisify(search); +export = search; + diff --git a/src/utils.js b/src/utils.js index fb59865f69..e9571253e9 100644 --- a/src/utils.js +++ b/src/utils.js @@ -4,19 +4,29 @@ const crypto = require('crypto'); const nconf = require('nconf'); const path = require('node:path'); -process.profile = function (operation, start) { - console.log('%s took %d milliseconds', operation, process.elapsedTimeSince(start)); -}; - -process.elapsedTimeSince = function (start) { +const elapsedTimeSince = function (start) { const diff = process.hrtime(start); return (diff[0] * 1e3) + (diff[1] / 1e6); }; + +process.profile = function (operation, start) { + console.log( + '%s took %d milliseconds', + operation, + process.elapsedTimeSince(start) + ); +}; + +process.elapsedTimeSince = elapsedTimeSince; const utils = { ...require('../public/src/utils.common') }; +utils.elapsedTimeSince = elapsedTimeSince; + utils.getLanguage = function () { const meta = require('./meta'); - return meta.config && meta.config.defaultLang ? meta.config.defaultLang : 'en-GB'; + return meta.config && meta.config.defaultLang ? + meta.config.defaultLang : + 'en-GB'; }; utils.generateUUID = function () { @@ -53,7 +63,7 @@ utils.getFontawesomePath = function () { utils.getFontawesomeStyles = function () { let styles = nconf.get('fontawesome:styles') || '*'; - // "*" is a special case, it means all styles, spread is used to support both string and array (["*"]) + // '*' is a special case, it means all styles, spread is used to support both string and array (['*']) if ([...styles][0] === '*') { styles = ['solid', 'brands', 'regular']; if (nconf.get('fontawesome:pro')) { diff --git a/test/search.js b/test/search.js index f0e285cb9d..b5cc0d3d98 100644 --- a/test/search.js +++ b/test/search.js @@ -1,6 +1,5 @@ 'use strict'; - const assert = require('assert'); const nconf = require('nconf'); @@ -28,21 +27,27 @@ describe('Search', () => { before(async () => { phoebeUid = await user.create({ username: 'phoebe' }); gingerUid = await user.create({ username: 'ginger' }); - cid1 = (await categories.create({ - name: 'Test Category', - description: 'Test category created by testing script', - })).cid; - - cid2 = (await categories.create({ - name: 'Test Category', - description: 'Test category created by testing script', - })).cid; - - cid3 = (await categories.create({ - name: 'Child Test Category', - description: 'Test category created by testing script', - parentCid: cid2, - })).cid; + cid1 = ( + await categories.create({ + name: 'Test Category', + description: 'Test category created by testing script', + }) + ).cid; + + cid2 = ( + await categories.create({ + name: 'Test Category', + description: 'Test category created by testing script', + }) + ).cid; + + cid3 = ( + await categories.create({ + name: 'Child Test Category', + description: 'Test category created by testing script', + parentCid: cid2, + }) + ).cid; ({ topicData: topic1Data, postData: post1Data } = await topics.post({ uid: phoebeUid, @@ -81,34 +86,32 @@ describe('Search', () => { await privileges.global.rescind(['groups:search:content'], 'guests'); }); - it('should search for a user', (done) => { - search.search({ - query: 'gin', - searchIn: 'users', - }, (err, data) => { - assert.ifError(err); + it('should search for a user', async () => { + try { + const data = await search.search({ + query: 'gin', + searchIn: 'users', + }); assert(data); assert.equal(data.matchCount, 1); assert.equal(data.users.length, 1); assert.equal(data.users[0].uid, gingerUid); assert.equal(data.users[0].username, 'ginger'); - done(); - }); + } catch (err) { + assert.ifError(err); + } }); - it('should search for a tag', (done) => { - search.search({ + it('should search for a tag', async () => { + const data = await search.search({ query: 'plug', searchIn: 'tags', - }, (err, data) => { - assert.ifError(err); - assert(data); - assert.equal(data.matchCount, 1); - assert.equal(data.tags.length, 1); - assert.equal(data.tags[0].value, 'plugin'); - assert.equal(data.tags[0].score, 2); - done(); }); + assert(data); + assert.equal(data.matchCount, 1); + assert.equal(data.tags.length, 1); + assert.equal(data.tags[0].value, 'plugin'); + assert.equal(data.tags[0].score, 2); }); it('should search for a category', async () => { @@ -130,55 +133,54 @@ describe('Search', () => { it('should search for categories', async () => { const socketCategories = require('../src/socket.io/categories'); - let data = await socketCategories.categorySearch({ uid: phoebeUid }, { query: 'baz', parentCid: 0 }); + let data = await socketCategories.categorySearch( + { uid: phoebeUid }, + { query: 'baz', parentCid: 0 } + ); assert.strictEqual(data[0].name, 'baz category'); - data = await socketCategories.categorySearch({ uid: phoebeUid }, { query: '', parentCid: 0 }); + data = await socketCategories.categorySearch( + { uid: phoebeUid }, + { query: '', parentCid: 0 } + ); assert.strictEqual(data.length, 5); }); - it('should fail if searchIn is wrong', (done) => { - search.search({ - query: 'plug', - searchIn: '', - }, (err) => { + it('should fail if searchIn is wrong', async () => { + try { + await search.search({ + query: 'plug', + searchIn: '', + }); + } catch (err) { assert.equal(err.message, '[[error:unknown-search-filter]]'); - done(); - }); + } }); - it('should search with tags filter', (done) => { - search.search({ + it('should search with tags filter', async () => { + const data = await search.search({ query: 'mongodb', searchIn: 'titles', hasTags: ['nodebb', 'javascript'], - }, (err, data) => { - assert.ifError(err); - assert.equal(data.posts[0].tid, topic2Data.tid); - done(); }); + assert.equal(data.posts[0].tid, topic2Data.tid); }); - it('should not crash if tags is not an array', (done) => { - search.search({ + it('should not crash if tags is not an array', async () => { + const data = await search.search({ query: 'mongodb', searchIn: 'titles', hasTags: 'nodebb,javascript', - }, (err, data) => { - assert.ifError(err); - done(); }); + assert(data); }); - it('should not find anything', (done) => { - search.search({ + it('should not find anything', async () => { + const data = await search.search({ query: 'xxxxxxxxxxxxxx', searchIn: 'titles', - }, (err, data) => { - assert.ifError(err); - assert(Array.isArray(data.posts)); - assert(!data.matchCount); - done(); }); + assert(Array.isArray(data.posts)); + assert(!data.matchCount); }); it('should search child categories', async () => { diff --git a/tsconfig.json b/tsconfig.json index 10aeeef7e6..9e8d61b6f7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,9 @@ "compilerOptions": { "allowJs": false, "target": "es6", - "module": "commonjs", + "module": "CommonJS", "moduleResolution": "node", - "esModuleInterop": true, + "esModuleInterop": true }, "include": [ "public/src/**/*", @@ -14,4 +14,4 @@ "exclude":[ "node_modules", ] -} \ No newline at end of file +}