From 63a21aabf8c40b5c29572add5dfe7e63bf569c21 Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Thu, 29 Jan 2026 14:54:53 -0500 Subject: [PATCH 1/7] Basic SearchClient, w/ some type & import cleanup --- integration-tests/admin/test/index.test.js | 103 +++--- modules/server/src/admin/index.ts | 163 +++++---- .../src/admin/schemas/AggsState/utils.ts | 110 +++--- .../src/admin/schemas/ColumnsState/utils.ts | 141 ++++---- .../admin/schemas/ExtendedMapping/utils.ts | 246 ++++++------- .../src/admin/schemas/IndexSchema/utils.ts | 342 +++++++++--------- .../src/admin/schemas/MatchboxState/utils.ts | 109 +++--- .../src/admin/schemas/ProjectSchema/utils.ts | 144 ++++---- .../src/admin/services/elasticsearch/index.ts | 45 +-- modules/server/src/admin/types.ts | 47 +-- modules/server/src/config/utils/index.ts | 5 +- modules/server/src/gqlServer.ts | 5 +- modules/server/src/mapping/utils/esSearch.ts | 6 +- .../server/src/mapping/utils/fetchMapping.ts | 7 +- modules/server/src/schema/Root.ts | 4 +- modules/server/src/searchClient/index.ts | 9 + modules/server/src/server.ts | 16 +- 17 files changed, 745 insertions(+), 757 deletions(-) create mode 100644 modules/server/src/searchClient/index.ts diff --git a/integration-tests/admin/test/index.test.js b/integration-tests/admin/test/index.test.js index 4ea0c5a84..b15aa2a91 100644 --- a/integration-tests/admin/test/index.test.js +++ b/integration-tests/admin/test/index.test.js @@ -1,9 +1,12 @@ +import { after, before, describe } from 'node:test'; + import { Client } from '@elastic/elasticsearch'; import express from 'express'; -import Arranger, { adminGraphql } from '../../../modules/server/dist'; -import ajax from '../../../modules/server/dist/utils/ajax'; -import addProject from './addProject'; +import Arranger, { adminGraphql } from '../../../modules/server/dist/index.js'; +import ajax from '../../../modules/server/dist/utils/ajax.js'; + +import addProject from './addProject.js'; const file_centric_mapppings = require('./assets/file_centric.mappings.json'); @@ -19,58 +22,58 @@ const app = express(); const api = ajax(`http://localhost:${port}`); const esClient = new Client({ - ...(useAuth && { - auth: { - username: esUser, - password: esPwd, - }, - }), - node: esHost, + ...(useAuth && { + auth: { + username: esUser, + password: esPwd, + }, + }), + node: esHost, }); const cleanup = () => - Promise.all([ - esClient.indices.delete({ - index: esIndex, - }), - esClient.indices.delete({ - index: 'arranger-projects*', - }), - ]); + Promise.all([ + esClient.indices.delete({ + index: esIndex, + }), + esClient.indices.delete({ + index: 'arranger-projects*', + }), + ]); describe('@arranger/admin', () => { - let server; - const adminPath = '/admin/graphql'; - before(async () => { - console.log('===== Initializing Elasticsearch data ====='); - try { - await cleanup(); - } catch (err) {} - await esClient.indices.create({ - index: esIndex, - body: file_centric_mapppings, - }); + let server; + const adminPath = '/admin/graphql'; + before(async () => { + console.log('===== Initializing Elasticsearch data ====='); + try { + await cleanup(); + } catch (err) {} + await esClient.indices.create({ + index: esIndex, + body: file_centric_mapppings, + }); - console.log('===== Starting arranger app for test ====='); - const router = await Arranger({ esHost, enableAdmin: false }); - const adminApp = await adminGraphql({ esHost }); - adminApp.applyMiddleware({ app, path: adminPath }); - app.use(router); - await new Promise((resolve) => { - server = app.listen(port, () => { - resolve(); - }); - }); - }); - after(async () => { - server?.close(); - await cleanup(); - }); + console.log('===== Starting arranger app for test ====='); + const router = await Arranger({ esHost, enableAdmin: false }); + const adminApp = await adminGraphql({ esHost }); + adminApp.applyMiddleware({ app, path: adminPath }); + app.use(router); + await new Promise((resolve) => { + server = app.listen(port, () => { + resolve(); + }); + }); + }); + after(async () => { + server?.close(); + await cleanup(); + }); - const env = { - api, - esIndex, - adminPath, - }; - addProject(env); + const env = { + api, + esIndex, + adminPath, + }; + addProject(env); }); diff --git a/modules/server/src/admin/index.ts b/modules/server/src/admin/index.ts index 956bb66cb..4b1b06df0 100644 --- a/modules/server/src/admin/index.ts +++ b/modules/server/src/admin/index.ts @@ -2,100 +2,99 @@ import { addMocksToSchema } from '@graphql-tools/mock'; import { mergeSchemas } from '@graphql-tools/schema'; import { ApolloServer } from 'apollo-server-express'; -import { Client } from '@elastic/elasticsearch'; +import { type GraphQLSchema } from 'graphql'; import { print } from 'graphql/language/printer'; -import { GraphQLSchema } from 'graphql'; + +import { type SearchClientType } from '../searchClient/index.js'; import { - createAggsStateByIndexResolver, - createColumnsStateByIndexResolver, - createExtendedMappingsByIndexResolver, - createIndexByProjectResolver, - createIndicesByProjectResolver, - createMatchBoxStateByIndexResolver, -} from './resolvers'; -import { createSchema as createProjectSchema } from './schemas/ProjectSchema'; -import { createSchema as createIndexSchema } from './schemas/IndexSchema'; -import { createSchema as createAggsStateSchema } from './schemas/AggsState'; -import { createSchema as createMatchboxStateSchema } from './schemas/MatchboxState'; -import { createSchema as createColumnsStateSchema } from './schemas/ColumnsState'; -import { createSchema as createExtendedMappingSchema } from './schemas/ExtendedMapping'; -import mergedTypeDefs from './schemaTypeDefs'; -import { constants } from './services/constants'; -import { createClient as createElasticsearchClient } from './services/elasticsearch'; -import { AdminApiConfig, IQueryContext } from './types'; + createAggsStateByIndexResolver, + createColumnsStateByIndexResolver, + createExtendedMappingsByIndexResolver, + createIndexByProjectResolver, + createIndicesByProjectResolver, + createMatchBoxStateByIndexResolver, +} from './resolvers.js'; +import { createSchema as createAggsStateSchema } from './schemas/AggsState/index.js'; +import { createSchema as createColumnsStateSchema } from './schemas/ColumnsState/index.js'; +import { createSchema as createExtendedMappingSchema } from './schemas/ExtendedMapping/index.js'; +import { createSchema as createIndexSchema } from './schemas/IndexSchema/index.js'; +import { createSchema as createMatchboxStateSchema } from './schemas/MatchboxState/index.js'; +import { createSchema as createProjectSchema } from './schemas/ProjectSchema/index.js'; +import mergedTypeDefs from './schemaTypeDefs.js'; +import { constants } from './services/constants.js'; +import { createClient as createElasticsearchClient } from './services/elasticsearch/index.js'; +import { type AdminApiConfig, type IQueryContext } from './types.js'; const createSchema = async () => { - const typeDefs = mergedTypeDefs; + const typeDefs = mergedTypeDefs; - const projectSchema = await createProjectSchema(); - const aggsStateSchema = await createAggsStateSchema(); - const collumnsStateSchema = await createColumnsStateSchema(); - const extendedMappingShema = await createExtendedMappingSchema(); - const matchBoxStateSchema = await createMatchboxStateSchema(); - const indexSchema = await createIndexSchema(); + const projectSchema = await createProjectSchema(); + const aggsStateSchema = await createAggsStateSchema(); + const collumnsStateSchema = await createColumnsStateSchema(); + const extendedMappingShema = await createExtendedMappingSchema(); + const matchBoxStateSchema = await createMatchboxStateSchema(); + const indexSchema = await createIndexSchema(); - const mergedSchema = mergeSchemas({ - schemas: [ - projectSchema, - indexSchema, - aggsStateSchema, - collumnsStateSchema, - extendedMappingShema, - matchBoxStateSchema, - print(typeDefs) as unknown as GraphQLSchema, // TODO: this type coercion is smelly - ], - resolvers: { - Project: { - index: createIndexByProjectResolver(indexSchema), - indices: createIndicesByProjectResolver(indexSchema), - }, - Index: { - extended: createExtendedMappingsByIndexResolver(extendedMappingShema), - columnsState: createColumnsStateByIndexResolver(collumnsStateSchema), - aggsState: createAggsStateByIndexResolver(aggsStateSchema), - matchBoxState: createMatchBoxStateByIndexResolver(matchBoxStateSchema), - }, - }, - }); - addMocksToSchema({ schema: mergedSchema, preserveResolvers: true }); - return mergedSchema; + const mergedSchema = mergeSchemas({ + schemas: [ + projectSchema, + indexSchema, + aggsStateSchema, + collumnsStateSchema, + extendedMappingShema, + matchBoxStateSchema, + print(typeDefs) as unknown as GraphQLSchema, // TODO: this type coercion is smelly + ], + resolvers: { + Project: { + index: createIndexByProjectResolver(indexSchema), + indices: createIndicesByProjectResolver(indexSchema), + }, + Index: { + extended: createExtendedMappingsByIndexResolver(extendedMappingShema), + columnsState: createColumnsStateByIndexResolver(collumnsStateSchema), + aggsState: createAggsStateByIndexResolver(aggsStateSchema), + matchBoxState: createMatchBoxStateByIndexResolver(matchBoxStateSchema), + }, + }, + }); + addMocksToSchema({ schema: mergedSchema, preserveResolvers: true }); + return mergedSchema; }; function buildElasticsearchClient(config: AdminApiConfig) { - return createElasticsearchClient(config.esHost, config.esUser, config.esPass); + return createElasticsearchClient(config.esHost, config.esUser, config.esPass); } -const initialize = (config: AdminApiConfig): Promise => - new Promise(async (resolve, reject) => { - console.info('Initializing Elasticsearch Client for host: ' + config.esHost); - const esClient = buildElasticsearchClient(config); - try { - console.info( - 'Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch', - ); - const exists = await esClient.indices.exists({ - index: constants.ARRANGER_PROJECT_INDEX, - }); - if (!exists) { - esClient.indices.create({ - index: constants.ARRANGER_PROJECT_INDEX, - }); - } - resolve(esClient); - } catch (err) { - setTimeout(() => { - initialize(config).then(() => resolve(esClient)); - }, 1000); - } - }); +const initialize = (config: AdminApiConfig): Promise => + new Promise(async (resolve, reject) => { + console.info('Initializing Elasticsearch Client for host: ' + config.esHost); + const esClient = buildElasticsearchClient(config); + try { + console.info('Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch'); + const exists = await esClient.indices.exists({ + index: constants.ARRANGER_PROJECT_INDEX, + }); + if (!exists) { + esClient.indices.create({ + index: constants.ARRANGER_PROJECT_INDEX, + }); + } + resolve(esClient); + } catch (err) { + setTimeout(() => { + initialize(config).then(() => resolve(esClient)); + }, 1000); + } + }); export default async (config: AdminApiConfig) => { - const esClient = await initialize(config); - return new ApolloServer({ - schema: await createSchema(), - context: (): IQueryContext => ({ - es: esClient, - }), - }); + const esClient = await initialize(config); + return new ApolloServer({ + schema: await createSchema(), + context: (): IQueryContext => ({ + es: esClient, + }), + }); }; diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index 63689eea7..3199ae4c9 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -1,67 +1,63 @@ -import { Client } from '@elastic/elasticsearch'; -import { mappingToAggsState } from '../../../mapping'; import { sortBy } from 'ramda'; + +import { mappingToAggsState } from '../../../mapping/index.js'; +import { type SearchClientType } from '../../../searchClient/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { timestamp } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_AggsSetState, - I_AggsState, - I_AggsStateQueryInput, - I_SaveAggsStateMutationInput, -} from './types'; -import { timestamp } from '../../services'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; -import { EsIndexLocation } from '../types'; -import { getEsMapping } from '../../services/elasticsearch'; + type I_AggsSetState, + type I_AggsState, + type I_AggsStateQueryInput, + type I_SaveAggsStateMutationInput, +} from './types.js'; export const createAggsSetState = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - const rawEsmapping = await getEsMapping(es)({ esIndex }); - const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; - const aggsState: I_AggsState[] = mappingToAggsState(mapping); - return { timestamp: timestamp(), state: aggsState }; - }; + (es: SearchClientType) => + async ({ esIndex }: EsIndexLocation): Promise => { + const rawEsmapping = await getEsMapping(es)({ esIndex }); + const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; + const aggsState: I_AggsState[] = mappingToAggsState(mapping); + return { timestamp: timestamp(), state: aggsState }; + }; export const getAggsSetState = - (es: Client) => - async (args: I_AggsStateQueryInput): Promise => { - const { projectId, graphqlField } = args; - const metaData = (await getProjectStorageMetadata(es)(projectId)).find( - (entry) => entry.name === graphqlField, - ); - return metaData.config['aggs-state']; - }; + (es: SearchClientType) => + async (args: I_AggsStateQueryInput): Promise => { + const { projectId, graphqlField } = args; + const metaData = (await getProjectStorageMetadata(es)(projectId)).find((entry) => entry.name === graphqlField); + return metaData.config['aggs-state']; + }; export const saveAggsSetState = - (es: Client) => - async (args: I_SaveAggsStateMutationInput): Promise => { - const { graphqlField, projectId, state } = args; - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - const currentAggsState = currentMetadata.config['aggs-state']; - const sortByNewOrder = sortBy((i: I_AggsState) => - state.findIndex((_i) => _i.field === i.field), - ); - const newAggsSetState: typeof currentAggsState = { - timestamp: timestamp(), - state: sortByNewOrder( - currentAggsState.state.map((item) => ({ - ...(state.find((_item) => _item.field === item.field) || item), - type: item.type, - })), - ), - }; + (es: SearchClientType) => + async (args: I_SaveAggsStateMutationInput): Promise => { + const { graphqlField, projectId, state } = args; + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + const currentAggsState = currentMetadata.config['aggs-state']; + const sortByNewOrder = sortBy((i: I_AggsState) => state.findIndex((_i) => _i.field === i.field)); + const newAggsSetState: typeof currentAggsState = { + timestamp: timestamp(), + state: sortByNewOrder( + currentAggsState.state.map((item) => ({ + ...(state.find((_item) => _item.field === item.field) || item), + type: item.type, + })), + ), + }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentMetadata.index, - name: currentMetadata.name, - config: { - 'aggs-state': newAggsSetState, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentMetadata.index, + name: currentMetadata.name, + config: { + 'aggs-state': newAggsSetState, + }, + }, + }); - return newAggsSetState; - }; + return newAggsSetState; + }; diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index ae366e06b..ace1b4771 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -1,80 +1,75 @@ -import { Client } from '@elastic/elasticsearch'; -import { - I_Column, - I_ColumnSetState, - I_ColumnStateQueryInput, - I_SaveColumnsStateMutationInput, -} from './types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; -import { EsIndexLocation } from '../types'; -import { mappingToColumnsState } from '../../../mapping'; -import { replaceBy, timestamp } from '../../services'; -import { getEsMapping } from '../../services/elasticsearch'; import { sortBy } from 'ramda'; +import { type SearchClientType } from '#searchClient/index.js'; + +import { mappingToColumnsState } from '../../../mapping/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { replaceBy, timestamp } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + +import { + type I_Column, + type I_ColumnSetState, + type I_ColumnStateQueryInput, + type I_SaveColumnsStateMutationInput, +} from './types.js'; + export const getColumnSetState = - (es: Client) => - async (args: I_ColumnStateQueryInput): Promise => { - const { graphqlField, projectId } = args; - const metaData = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - return metaData.config['columns-state']; - }; + (es: SearchClientType) => + async (args: I_ColumnStateQueryInput): Promise => { + const { graphqlField, projectId } = args; + const metaData = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + return metaData.config['columns-state']; + }; export const createColumnSetState = - (es: Client) => - async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { - const rawEsmapping = await getEsMapping(es)({ - esIndex, - }); - const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings; - const columns: I_Column[] = mappingToColumnsState(mapping.properties); - return { - state: { - type: graphqlField, - keyField: 'id', - defaultSorted: [{ id: columns[0].id || columns[0].accessor, desc: false }], - columns, - }, - timestamp: timestamp(), - }; - }; + (es: SearchClientType) => + async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { + const rawEsmapping = await getEsMapping(es)({ + esIndex, + }); + const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings; + const columns: I_Column[] = mappingToColumnsState(mapping.properties); + return { + state: { + type: graphqlField, + keyField: 'id', + defaultSorted: [{ id: columns[0].id || columns[0].accessor, desc: false }], + columns, + }, + timestamp: timestamp(), + }; + }; export const saveColumnState = - (es: Client) => - async ({ - graphqlField, - projectId, - state, - }: I_SaveColumnsStateMutationInput): Promise => { - const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); - const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); - const sortByNewOrder = sortBy((i: I_Column) => - state.columns.findIndex((c) => c.field === i.field), - ); - const mergedState: typeof state = { - ...state, - columns: sortByNewOrder( - replaceBy( - currentIndexMetadata.config['columns-state'].state.columns, - state.columns, - (oldCol, newCol) => oldCol.field === newCol.field, - ), - ), - }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - 'columns-state': { - timestamp: timestamp(), - state: mergedState, - }, - }, - }, - }); - return getColumnSetState(es)({ projectId, graphqlField }); - }; + (es: SearchClientType) => + async ({ graphqlField, projectId, state }: I_SaveColumnsStateMutationInput): Promise => { + const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); + const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); + const sortByNewOrder = sortBy((i: I_Column) => state.columns.findIndex((c) => c.field === i.field)); + const mergedState: typeof state = { + ...state, + columns: sortByNewOrder( + replaceBy( + currentIndexMetadata.config['columns-state'].state.columns, + state.columns, + (oldCol, newCol) => oldCol.field === newCol.field, + ), + ), + }; + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata.index, + name: currentIndexMetadata.name, + config: { + 'columns-state': { + timestamp: timestamp(), + state: mergedState, + }, + }, + }, + }); + return getColumnSetState(es)({ projectId, graphqlField }); + }; diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index 6773ee9c4..038ce9e3c 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -1,145 +1,137 @@ -import { Client } from '@elastic/elasticsearch'; import { UserInputError } from 'apollo-server'; -import { extendMapping } from '../../../mapping'; -import { getEsMapping } from '../../services/elasticsearch'; -import { replaceBy } from '../../services'; -import { EsIndexLocation } from '../types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; +import { type SearchClientType } from '#searchClient/index.js'; + +import { extendMapping } from '../../../mapping/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { replaceBy } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_ExtendedFieldsMappingsQueryArgs, - I_GqlExtendedFieldMapping, - I_SaveExtendedMappingMutationArgs, - I_UpdateExtendedMappingMutationArgs, -} from './types'; + type I_ExtendedFieldsMappingsQueryArgs, + type I_GqlExtendedFieldMapping, + type I_SaveExtendedMappingMutationArgs, + type I_UpdateExtendedMappingMutationArgs, +} from './types.js'; export const createExtendedMapping = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - let extendedMappings: I_GqlExtendedFieldMapping[] = []; - try { - const esMapping = await getEsMapping(es)({ esIndex }); - const indexName = Object.keys(esMapping)[0]; //assumes all mappings returned are the same - const esMappingProperties = esMapping[indexName].mappings.properties; - extendedMappings = extendMapping(esMappingProperties) as I_GqlExtendedFieldMapping[]; - } catch (err) { - console.log('error: ', err); - throw err; - } - return extendedMappings; - }; + (es: SearchClientType) => + async ({ esIndex }: EsIndexLocation): Promise => { + let extendedMappings: I_GqlExtendedFieldMapping[] = []; + try { + const esMapping = await getEsMapping(es)({ esIndex }); + const indexName = Object.keys(esMapping)[0]; //assumes all mappings returned are the same + const esMappingProperties = esMapping[indexName].mappings.properties; + extendedMappings = extendMapping(esMappingProperties) as I_GqlExtendedFieldMapping[]; + } catch (err) { + console.log('error: ', err); + throw err; + } + return extendedMappings; + }; export const getExtendedMapping = - (es: Client) => - async ({ - projectId, - graphqlField, - field, - }: I_ExtendedFieldsMappingsQueryArgs): Promise => { - const assertOutputType = (i: any): I_GqlExtendedFieldMapping => ({ - gqlId: `${projectId}::${graphqlField}::extended::${i.field}`, - field: i.field, - type: i.type, - displayName: i.displayName, - active: i.active, - isArray: i.isArray, - primaryKey: i.primaryKey, - quickSearchEnabled: i.quickSearchEnabled, - unit: i.unit, - displayValues: i.displayValues, - rangeStep: i.rangeStep, - }); - const indexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (metaData) => metaData.name === graphqlField, - ); - if (indexMetadata) { - if (field) { - return indexMetadata.config.extended - .filter((ex) => field === ex.field) - .map(assertOutputType); - } else { - return indexMetadata.config.extended.map(assertOutputType); - } - } else { - throw new UserInputError( - `no index found under name ${graphqlField} for project ${projectId}`, - ); - } - }; + (es: SearchClientType) => + async ({ + projectId, + graphqlField, + field, + }: I_ExtendedFieldsMappingsQueryArgs): Promise => { + const assertOutputType = (i: any): I_GqlExtendedFieldMapping => ({ + gqlId: `${projectId}::${graphqlField}::extended::${i.field}`, + field: i.field, + type: i.type, + displayName: i.displayName, + active: i.active, + isArray: i.isArray, + primaryKey: i.primaryKey, + quickSearchEnabled: i.quickSearchEnabled, + unit: i.unit, + displayValues: i.displayValues, + rangeStep: i.rangeStep, + }); + const indexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( + (metaData) => metaData.name === graphqlField, + ); + if (indexMetadata) { + if (field) { + return indexMetadata.config.extended.filter((ex) => field === ex.field).map(assertOutputType); + } else { + return indexMetadata.config.extended.map(assertOutputType); + } + } else { + throw new UserInputError(`no index found under name ${graphqlField} for project ${projectId}`); + } + }; export const updateFieldExtendedMapping = - (es: Client) => - async ({ - field: mutatedField, - graphqlField, - projectId, - extendedFieldMappingInput, - }: I_UpdateExtendedMappingMutationArgs): Promise => { - const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (metaData) => { - return metaData.name === graphqlField; - }, - ); + (es: SearchClientType) => + async ({ + field: mutatedField, + graphqlField, + projectId, + extendedFieldMappingInput, + }: I_UpdateExtendedMappingMutationArgs): Promise => { + const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find((metaData) => { + return metaData.name === graphqlField; + }); - if (currentIndexMetadata) { - const indexExtendedMappingFields = await getExtendedMapping(es)({ - projectId, - graphqlField, - }); + if (currentIndexMetadata) { + const indexExtendedMappingFields = await getExtendedMapping(es)({ + projectId, + graphqlField, + }); - const newIndexExtendedMappingFields: I_GqlExtendedFieldMapping[] = - indexExtendedMappingFields.map((field) => - (field.field as string) === mutatedField - ? { ...field, ...extendedFieldMappingInput } - : field, - ); + const newIndexExtendedMappingFields: I_GqlExtendedFieldMapping[] = indexExtendedMappingFields.map( + (field) => + (field.field as string) === mutatedField ? { ...field, ...extendedFieldMappingInput } : field, + ); - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - extended: newIndexExtendedMappingFields, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata.index, + name: currentIndexMetadata.name, + config: { + extended: newIndexExtendedMappingFields, + }, + }, + }); - return newIndexExtendedMappingFields.find((field) => field.field === mutatedField); - } else { - throw new UserInputError( - `no index found under name ${graphqlField} for project ${projectId}`, - ); - } - }; + return newIndexExtendedMappingFields.find((field) => field.field === mutatedField); + } else { + throw new UserInputError(`no index found under name ${graphqlField} for project ${projectId}`); + } + }; export const saveExtendedMapping = - (es: Client) => - async (args: I_SaveExtendedMappingMutationArgs): Promise => { - const { projectId, graphqlField, input } = args; - const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (entry) => entry.name === graphqlField, - ); - const { - config: { extended: currentStoredExtendedMapping }, - } = currentIndexMetadata; + (es: SearchClientType) => + async (args: I_SaveExtendedMappingMutationArgs): Promise => { + const { projectId, graphqlField, input } = args; + const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( + (entry) => entry.name === graphqlField, + ); + const { + config: { extended: currentStoredExtendedMapping }, + } = currentIndexMetadata; - const newExtendedMapping: I_GqlExtendedFieldMapping[] = replaceBy( - currentStoredExtendedMapping, - input, - (el1, el2) => el1.field === el2.field, - ); + const newExtendedMapping: I_GqlExtendedFieldMapping[] = replaceBy( + currentStoredExtendedMapping, + input, + (el1, el2) => el1.field === el2.field, + ); - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - extended: newExtendedMapping, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata.index, + name: currentIndexMetadata.name, + config: { + extended: newExtendedMapping, + }, + }, + }); - return newExtendedMapping; - }; + return newExtendedMapping; + }; diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index e310a92b6..4686c9585 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -1,203 +1,201 @@ -import { Client } from '@elastic/elasticsearch'; import { UserInputError } from 'apollo-server'; import Qew from 'qew'; // TODO: using 0.9.13 because later versions break the async -import { getEsMapping } from '../../services/elasticsearch'; -import { constants } from '../../services/constants'; -import { serializeToGqlField, timestamp } from '../../services'; -import { createExtendedMapping } from '../ExtendedMapping/utils'; -import { getArrangerProjects } from '../ProjectSchema/utils'; -import { EsIndexLocation } from '../types'; +import { type SearchClientType } from '#searchClient/index.js'; + +import { constants } from '../../services/constants.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { serializeToGqlField, timestamp } from '../../services/index.js'; +import { createAggsSetState } from '../AggsState/utils.js'; +import { createColumnSetState } from '../ColumnsState/utils.js'; +import { createExtendedMapping } from '../ExtendedMapping/utils.js'; +import { createMatchboxState } from '../MatchboxState/utils.js'; +import { getArrangerProjects } from '../ProjectSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_ProjectIndexMetadataUpdateDoc, - IIndexGqlModel, - IIndexQueryInput, - IIndexRemovalMutationInput, - INewIndexInput, - IProjectIndexMetadata, -} from './types'; -import { createColumnSetState } from '../ColumnsState/utils'; -import { createAggsSetState } from '../AggsState/utils'; -import { createMatchboxState } from '../MatchboxState/utils'; + type I_ProjectIndexMetadataUpdateDoc, + type IIndexGqlModel, + type IIndexQueryInput, + type IIndexRemovalMutationInput, + type INewIndexInput, + type IProjectIndexMetadata, +} from './types.js'; const { ARRANGER_PROJECT_INDEX } = constants; export const getProjectMetadataEsLocation = ( - projectId: string, + projectId: string, ): { - index: string; + index: string; } => ({ - index: `${ARRANGER_PROJECT_INDEX}-${projectId}`, + index: `${ARRANGER_PROJECT_INDEX}-${projectId}`, }); const mappingExistsOn = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - try { - await getEsMapping(es)({ esIndex }); - return true; - } catch (err) { - return false; - } - }; + (es: SearchClientType) => + async ({ esIndex }: EsIndexLocation): Promise => { + try { + await getEsMapping(es)({ esIndex }); + return true; + } catch (err) { + return false; + } + }; export const getProjectStorageMetadata = - (es: Client) => - async (projectId: string): Promise => { - try { - const { - body: { - hits: { hits }, - }, - } = await es.search({ - ...getProjectMetadataEsLocation(projectId), - }); - return hits.map(({ _source }) => _source as IProjectIndexMetadata); - } catch (err) { - throw new UserInputError(`cannot find project of id ${projectId}`, err); - } - }; + (es: SearchClientType) => + async (projectId: string): Promise => { + try { + const { + body: { + hits: { hits }, + }, + } = await es.search({ + ...getProjectMetadataEsLocation(projectId), + }); + return hits.map(({ _source }) => _source as IProjectIndexMetadata); + } catch (err) { + throw new UserInputError(`cannot find project of id ${projectId}`, err); + } + }; export const getProjectMetadata = - (es: Client) => - async (projectId: string): Promise => - Promise.all( - (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ - id: `${projectId}::${metadata.name}`, - hasMapping: mappingExistsOn(es)({ - esIndex: metadata.index, - }), - graphqlField: metadata.name, - projectId: projectId, - esIndex: metadata.index, - })), - ); + (es: SearchClientType) => + async (projectId: string): Promise => + Promise.all( + (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ + id: `${projectId}::${metadata.name}`, + hasMapping: mappingExistsOn(es)({ + esIndex: metadata.index, + }), + graphqlField: metadata.name, + projectId: projectId, + esIndex: metadata.index, + })), + ); export const createNewIndex = - (es: Client) => - async (args: INewIndexInput): Promise => { - const { projectId, graphqlField, esIndex } = args; - const arrangerProject: {} = (await getArrangerProjects(es)).find( - (project) => project.id === projectId, - ); - if (arrangerProject) { - const serializedGqlField = serializeToGqlField(graphqlField); - - const extendedMapping = await createExtendedMapping(es)({ - esIndex, - }); - - const metadataContent: IProjectIndexMetadata = { - index: esIndex, - name: serializedGqlField, - timestamp: timestamp(), - active: true, - config: { - 'aggs-state': await createAggsSetState(es)({ esIndex }), - 'columns-state': await createColumnSetState(es)( - { - esIndex, - }, - graphqlField, - ), - 'matchbox-state': createMatchboxState({ - graphqlField, - extendedFields: extendedMapping, - }), - extended: extendedMapping, - }, - }; - - await es.create({ - ...getProjectMetadataEsLocation(projectId), - id: esIndex, - body: metadataContent, - refresh: 'true', - }); - - return getProjectIndex(es)({ - projectId, - graphqlField: serializedGqlField, - }); - } else { - throw new UserInputError(`no project with ID ${projectId} was found`); - } - }; + (es: SearchClientType) => + async (args: INewIndexInput): Promise => { + const { projectId, graphqlField, esIndex } = args; + const arrangerProject: {} = (await getArrangerProjects(es)).find((project) => project.id === projectId); + if (arrangerProject) { + const serializedGqlField = serializeToGqlField(graphqlField); + + const extendedMapping = await createExtendedMapping(es)({ + esIndex, + }); + + const metadataContent: IProjectIndexMetadata = { + index: esIndex, + name: serializedGqlField, + timestamp: timestamp(), + active: true, + config: { + 'aggs-state': await createAggsSetState(es)({ esIndex }), + 'columns-state': await createColumnSetState(es)( + { + esIndex, + }, + graphqlField, + ), + 'matchbox-state': createMatchboxState({ + graphqlField, + extendedFields: extendedMapping, + }), + extended: extendedMapping, + }, + }; + + await es.create({ + ...getProjectMetadataEsLocation(projectId), + id: esIndex, + body: metadataContent, + refresh: 'true', + }); + + return getProjectIndex(es)({ + projectId, + graphqlField: serializedGqlField, + }); + } else { + throw new UserInputError(`no project with ID ${projectId} was found`); + } + }; // because different metadata entities write to the same ES document, update operations need to be queued up to a single concurrency controlled task queue for each project. This factory creates a task queue manager for this purpose. const createProjectQueueManager = () => { - const queues: { - [projectId: string]: any; - } = {}; - return { - getQueue: (projectId: string) => { - if (!queues[projectId]) { - queues[projectId] = new Qew(); - } - return queues[projectId]; - }, - }; + const queues: { + [projectId: string]: any; + } = {}; + return { + getQueue: (projectId: string) => { + if (!queues[projectId]) { + queues[projectId] = new Qew(); + } + return queues[projectId]; + }, + }; }; // pretty bad, since we're just taking anything right now in run time, but at least graphQl will ensure `metaData` is typed in runtime const projectQueueManager = createProjectQueueManager(); export const updateProjectIndexMetadata = - (es: Client) => - async ({ - projectId, - metaData, - }: { - projectId: string; - metaData: I_ProjectIndexMetadataUpdateDoc; - }): Promise => { - const queue = projectQueueManager.getQueue(projectId); - - return queue.pushProm(async () => { - await es.update({ - ...getProjectMetadataEsLocation(projectId), - id: metaData.index, - body: { - doc: metaData, - }, - refresh: 'true', - }); - - const output = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === metaData.name, - ); - - return output; - }); - }; + (es: Client) => + async ({ + projectId, + metaData, + }: { + projectId: string; + metaData: I_ProjectIndexMetadataUpdateDoc; + }): Promise => { + const queue = projectQueueManager.getQueue(projectId); + + return queue.pushProm(async () => { + await es.update({ + ...getProjectMetadataEsLocation(projectId), + id: metaData.index, + body: { + doc: metaData, + }, + refresh: 'true', + }); + + const output = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === metaData.name); + + return output; + }); + }; export const getProjectIndex = - (es: Client) => - async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { - try { - const output = (await getProjectMetadata(es)(projectId)).find( - ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, - ); - return output; - } catch { - throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); - } - }; + (es: SearchClientType) => + async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { + try { + const output = (await getProjectMetadata(es)(projectId)).find( + ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, + ); + return output; + } catch { + throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); + } + }; export const removeProjectIndex = - (es: Client) => - async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { - try { - const removedIndexMetadata = await getProjectIndex(es)({ - projectId, - graphqlField, - }); - await es.delete({ - ...getProjectMetadataEsLocation(projectId), - id: removedIndexMetadata.esIndex as string, - refresh: 'true', - }); - return removedIndexMetadata; - } catch (err) { - throw new UserInputError(`could not remove index ${graphqlField} of project ${projectId}`); - } - }; + (es: SearchClientType) => + async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { + try { + const removedIndexMetadata = await getProjectIndex(es)({ + projectId, + graphqlField, + }); + await es.delete({ + ...getProjectMetadataEsLocation(projectId), + id: removedIndexMetadata.esIndex as string, + refresh: 'true', + }); + return removedIndexMetadata; + } catch (err) { + throw new UserInputError(`could not remove index ${graphqlField} of project ${projectId}`); + } + }; diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 1e1264ee4..44fca9878 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -1,69 +1,66 @@ -import { Client } from '@elastic/elasticsearch'; +import { type SearchClientType } from '#searchClient/index.js'; + +import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping/index.js'; +import { replaceBy, timestamp } from '../../services/index.js'; +import { type I_GqlExtendedFieldMapping } from '../ExtendedMapping/types.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; -import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping'; -import { replaceBy, timestamp } from '../../services'; -import { I_GqlExtendedFieldMapping } from '../ExtendedMapping/types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; import { - I_MatchBoxField, - I_MatchBoxState, - I_MatchBoxStateQueryInput, - I_SaveMatchBoxStateMutationInput, -} from './types'; + type I_MatchBoxField, + type I_MatchBoxState, + type I_MatchBoxStateQueryInput, + type I_SaveMatchBoxStateMutationInput, +} from './types.js'; export const createMatchboxState = ({ - extendedFields, - graphqlField, + extendedFields, + graphqlField, }: { - extendedFields: Array; - graphqlField: string; + extendedFields: I_GqlExtendedFieldMapping[]; + graphqlField: string; }): I_MatchBoxState => { - const fields: I_MatchBoxField[] = extendedFieldsToMatchBoxState({ - extendedFields, - name: graphqlField, - }); - return { state: fields, timestamp: timestamp() }; + const fields: I_MatchBoxField[] = extendedFieldsToMatchBoxState({ + extendedFields, + name: graphqlField, + }); + return { state: fields, timestamp: timestamp() }; }; export const getMatchBoxState = - (es: Client) => - async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - return currentMetadata.config['matchbox-state']; - }; + (es: SearchClientType) => + async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + return currentMetadata.config['matchbox-state']; + }; export const saveMatchBoxState = - (es: Client) => - async ({ - graphqlField, - projectId, - state: updatedMatchboxFields, - }: I_SaveMatchBoxStateMutationInput): Promise => { - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - const currentMatchboxFields = currentMetadata.config['matchbox-state'].state; - const newMatchboxState: I_MatchBoxState = { - timestamp: timestamp(), - state: replaceBy( - currentMatchboxFields, - updatedMatchboxFields, - ({ field: field1 }, { field: field2 }) => field1 === field2, - ), - }; + (es: SearchClientType) => + async ({ + graphqlField, + projectId, + state: updatedMatchboxFields, + }: I_SaveMatchBoxStateMutationInput): Promise => { + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + const currentMatchboxFields = currentMetadata.config['matchbox-state'].state; + const newMatchboxState: I_MatchBoxState = { + timestamp: timestamp(), + state: replaceBy( + currentMatchboxFields, + updatedMatchboxFields, + ({ field: field1 }, { field: field2 }) => field1 === field2, + ), + }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentMetadata.index, - name: currentMetadata.name, - config: { - 'matchbox-state': newMatchboxState, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentMetadata.index, + name: currentMetadata.name, + config: { + 'matchbox-state': newMatchboxState, + }, + }, + }); - return newMatchboxState; - }; + return newMatchboxState; + }; diff --git a/modules/server/src/admin/schemas/ProjectSchema/utils.ts b/modules/server/src/admin/schemas/ProjectSchema/utils.ts index d5a06bdcf..7ba000041 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/utils.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/utils.ts @@ -1,82 +1,84 @@ -import { Client } from '@elastic/elasticsearch'; -import { constants } from '../../services/constants'; -import { serializeToEsId } from '../../services'; -import { IArrangerProject } from './types'; -import { getProjectMetadataEsLocation } from '../IndexSchema/utils'; +import { type SearchClientType } from '#searchClient/index.js'; + +import { constants } from '../../services/constants.js'; +import { serializeToEsId } from '../../services/index.js'; +import { getProjectMetadataEsLocation } from '../IndexSchema/utils.js'; + +import { type IArrangerProject } from './types.js'; const { ARRANGER_PROJECT_INDEX } = constants; export const newArrangerProject = (id: string): IArrangerProject => ({ - id: serializeToEsId(id), - active: true, - timestamp: new Date().toISOString(), + id: serializeToEsId(id), + active: true, + timestamp: new Date().toISOString(), }); -export const getArrangerProjects = async (es: Client): Promise> => { - const { - body: { - hits: { hits }, - }, - }: { - body: { - hits: { - hits: Array<{ - _source: any; - }>; - }; - }; - } = await es - .search({ - index: ARRANGER_PROJECT_INDEX, - }) - .catch(() => ({ - body: { - hits: { - hits: [], - }, - }, - })); - return hits.map(({ _source }) => _source as IArrangerProject); +export const getArrangerProjects = async (es: SearchClientType): Promise => { + const { + body: { + hits: { hits }, + }, + }: { + body: { + hits: { + hits: { + _source: any; + }[]; + }; + }; + } = await es + .search({ + index: ARRANGER_PROJECT_INDEX, + }) + .catch(() => ({ + body: { + hits: { + hits: [], + }, + }, + })); + return hits.map(({ _source }) => _source as IArrangerProject); }; export const addArrangerProject = - (es: Client) => - async (id: string): Promise => { - //id must be lower case - const _id = serializeToEsId(id); - const newProject = newArrangerProject(_id); - await Promise.all([ - await es.indices.create({ index: getProjectMetadataEsLocation(id).index }), - await es - .create({ - index: ARRANGER_PROJECT_INDEX, - id: _id, - body: newProject, - refresh: 'true', - }) - .then(() => newProject) - .catch(Promise.reject), - ]); - return getArrangerProjects(es); - }; + (es: SearchClientType) => + async (id: string): Promise => { + //id must be lower case + const _id = serializeToEsId(id); + const newProject = newArrangerProject(_id); + await Promise.all([ + await es.indices.create({ index: getProjectMetadataEsLocation(id).index }), + await es + .create({ + index: ARRANGER_PROJECT_INDEX, + id: _id, + body: newProject, + refresh: 'true', + }) + .then(() => newProject) + .catch(Promise.reject), + ]); + return getArrangerProjects(es); + }; export const removeArrangerProject = - (es: Client) => - async (id: string): Promise => { - const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); - if (existingProject) { - await Promise.all([ - es.indices.delete({ - index: getProjectMetadataEsLocation(id).index, - }), - es.delete({ - index: ARRANGER_PROJECT_INDEX, - id: id, - refresh: 'true', - }), - ]); - return getArrangerProjects(es); - } else { - return Promise.reject(`No project with id ${id} was found`); - } - }; + (es: SearchClientType) => + async (id: string): Promise => { + const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); + if (existingProject) { + await Promise.all([ + es.indices.delete({ + index: getProjectMetadataEsLocation(id).index, + }), + es.delete({ + index: ARRANGER_PROJECT_INDEX, + id: id, + refresh: 'true', + }), + ]); + return getArrangerProjects(es); + } else { + return Promise.reject(`No project with id ${id} was found`); + } + }; diff --git a/modules/server/src/admin/services/elasticsearch/index.ts b/modules/server/src/admin/services/elasticsearch/index.ts index 176faa353..90f3a4206 100644 --- a/modules/server/src/admin/services/elasticsearch/index.ts +++ b/modules/server/src/admin/services/elasticsearch/index.ts @@ -1,27 +1,28 @@ -import { Client } from '@elastic/elasticsearch'; -import { EsMapping } from './types'; +import SearchClient, { type SearchClientType } from '#searchClient/index.js'; + +import { type EsMapping } from './types.js'; export const createClient = (esHost: string, esUser: string, esPass: string) => { - const esConf = { node: esHost }; - if (esUser && esPass) { - esConf['auth'] = { - username: esUser, - password: esPass, - }; - } - return new Client(esConf); + const esConf = { node: esHost }; + if (esUser && esPass) { + esConf['auth'] = { + username: esUser, + password: esPass, + }; + } + return SearchClient(esConf); }; export const getEsMapping = - (es: Client) => - async ({ - esIndex, - }: { - esIndex: string; - esType?: string; //deprecated - }): Promise => { - const response = await es.indices.getMapping({ - index: esIndex, - }); - return response.body as EsMapping; - }; + (es: SearchClientType) => + async ({ + esIndex, + }: { + esIndex: string; + esType?: string; //deprecated + }): Promise => { + const response = await es.indices.getMapping({ + index: esIndex, + }); + return response.body as EsMapping; + }; diff --git a/modules/server/src/admin/types.ts b/modules/server/src/admin/types.ts index c697b1a7a..ce5c6d497 100644 --- a/modules/server/src/admin/types.ts +++ b/modules/server/src/admin/types.ts @@ -1,28 +1,29 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { MergeInfo } from 'graphql-tools'; -import { Client } from '@elastic/elasticsearch'; +import { type GraphQLResolveInfo } from 'graphql'; +import { type MergeInfo } from 'graphql-tools'; -export interface AdminApiConfig { - esHost: string; - esUser: string; - esPass: string; -} -export interface IQueryContext { - es: Client; -} +import { type SearchClientType } from '../searchClient/index.js'; + +export type AdminApiConfig = { + esHost: string; + esUser: string; + esPass: string; +}; +export type IQueryContext = { + es: SearchClientType; +}; export type ResolverOutput = T | Promise; -export type MergeResolver = - | (( - a: any, - args: Args, - c: IQueryContext, - d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, - ) => ResolverOutput) - | ResolverOutput; +export type MergeResolver = + | (( + a: any, + args: Args, + c: IQueryContext, + d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, + ) => ResolverOutput) + | ResolverOutput; -export interface I_MergeSchema { - fragment: string; - resolve: MergeResolver; -} +export type I_MergeSchema = { + fragment: string; + resolve: MergeResolver; +}; diff --git a/modules/server/src/config/utils/index.ts b/modules/server/src/config/utils/index.ts index 66d38186e..6166bdea7 100644 --- a/modules/server/src/config/utils/index.ts +++ b/modules/server/src/config/utils/index.ts @@ -1,14 +1,13 @@ -import type { Client } from '@elastic/elasticsearch'; - import { ENV_CONFIG } from '#config/index.js'; import { type ConfigObject, configProperties } from '#config/types.js'; import { setsMapping } from '#schema/index.js'; +import { type SearchClientType } from '#searchClient/index.js'; export const initializeSets = async ({ esClient, setsIndex: setsIndexParam, }: { - esClient: Client; + esClient: SearchClientType; setsIndex: string; }): Promise => { ENV_CONFIG.DEBUG_MODE && console.log(`Attempting to create Sets index "${setsIndexParam}"...`); diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index 65afa7e70..5e4ec62e9 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -1,8 +1,9 @@ -import { Client } from '@elastic/elasticsearch'; import { type GraphQLResolveInfo } from 'graphql'; +import { type SearchClientType } from './searchClient/index.js'; + export type Context = { - esClient: Client; + esClient: SearchClientType; }; export type Root = Record; diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts index 659a70b51..82b4f16e3 100644 --- a/modules/server/src/mapping/utils/esSearch.ts +++ b/modules/server/src/mapping/utils/esSearch.ts @@ -1,5 +1,7 @@ -import { Client, type RequestParams } from '@elastic/elasticsearch'; +import { type RequestParams } from '@elastic/elasticsearch'; -export default (esClient: Client) => async (params: RequestParams.Search) => { +import { type SearchClientType } from '#searchClient/index.js'; + +export default (esClient: SearchClientType) => async (params: RequestParams.Search) => { return (await esClient?.search(params))?.body; }; diff --git a/modules/server/src/mapping/utils/fetchMapping.ts b/modules/server/src/mapping/utils/fetchMapping.ts index 3cddb929b..a6dce4f8f 100644 --- a/modules/server/src/mapping/utils/fetchMapping.ts +++ b/modules/server/src/mapping/utils/fetchMapping.ts @@ -1,7 +1,8 @@ -import type { Client } from '@elastic/elasticsearch/api/new'; import type { CatAliasesAliasesRecord } from '@elastic/elasticsearch/api/types'; -export const getESAliases = async (esClient: Client) => { +import { type SearchClientType } from '#searchClient/index.js'; + +export const getESAliases = async (esClient: SearchClientType) => { const { body } = await esClient.cat.aliases({ format: 'json' }); return body; @@ -10,7 +11,7 @@ export const getESAliases = async (esClient: Client) => { export const checkESAlias = (aliases: CatAliasesAliasesRecord[], possibleAlias: string) => aliases?.find((foundIndex = { alias: undefined }) => foundIndex.alias === possibleAlias)?.index; -export const fetchMapping = async ({ esClient, index }: { esClient: Client; index: string }) => { +export const fetchMapping = async ({ esClient, index }: { esClient: SearchClientType; index: string }) => { if (esClient) { console.log(`Fetching ES mapping for "${index}"...`); const aliases = await getESAliases(esClient); diff --git a/modules/server/src/schema/Root.ts b/modules/server/src/schema/Root.ts index 9e93626c4..363f675ba 100644 --- a/modules/server/src/schema/Root.ts +++ b/modules/server/src/schema/Root.ts @@ -1,4 +1,3 @@ -import type { Client } from '@elastic/elasticsearch/api/new'; import { GraphQLDate } from 'graphql-scalars'; import { GraphQLJSON } from 'graphql-type-json'; import { startCase } from 'lodash-es'; @@ -7,6 +6,7 @@ import Parallel from 'paralleljs'; import { ENV_CONFIG } from '#config/index.js'; import { createConnectionResolvers, saveSet, mappingToFields } from '#mapping/index.js'; import { checkESAlias, getESAliases } from '#mapping/utils/fetchMapping.js'; +import { type SearchClientType } from '#searchClient/index.js'; import { typeDefs as AggregationsTypeDefs } from './Aggregations.js'; import ConfigsTypeDefs from './configQuery.js'; @@ -82,7 +82,7 @@ export const resolvers = ({ enableAdmin, types, rootTypes, scalarTypes, getServe Date: GraphQLDate, Root: { viewer: resolveObject, - hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: Client }) => { + hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: SearchClientType }) => { if (documentType) { if (index) { const [_, type] = types.find(([name]) => name === documentType) || []; diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts new file mode 100644 index 000000000..a1cbe93c1 --- /dev/null +++ b/modules/server/src/searchClient/index.ts @@ -0,0 +1,9 @@ +import { Client, type ClientOptions } from '@elastic/elasticsearch'; + +const SearchClient = (options: ClientOptions) => { + return new Client(options); +}; + +export type SearchClientType = ReturnType; + +export default SearchClient; diff --git a/modules/server/src/server.ts b/modules/server/src/server.ts index 131e49150..035ff9d99 100644 --- a/modules/server/src/server.ts +++ b/modules/server/src/server.ts @@ -1,4 +1,4 @@ -import { Client, type ClientOptions } from '@elastic/elasticsearch'; +import { type ClientOptions } from '@elastic/elasticsearch'; import { Router } from 'express'; import morgan from 'morgan'; @@ -6,18 +6,10 @@ import { ENABLE_LOGS, ES_ARRANGER_SET_INDEX, ENABLE_NETWORK_AGGREGATION } from ' import { ENV_CONFIG } from './config/index.js'; import downloadRoutes from './download/index.js'; import getGraphQLRoutes from './graphqlRoutes.js'; +import SearchClient from './searchClient/index.js'; import getDefaultServerSideFilter from './utils/getDefaultServerSideFilter.js'; -const { - CONFIG_FILES_PATH, - DEBUG_MODE, - ENABLE_ADMIN, - ES_HOST, - ES_USER, - ES_PASS, - ES_LOG, //TODO: ES doesn't include a logger anymore - PING_PATH, -} = ENV_CONFIG; +const { CONFIG_FILES_PATH, DEBUG_MODE, ENABLE_ADMIN, ES_HOST, ES_USER, ES_PASS, PING_PATH } = ENV_CONFIG; export const buildEsClient = (esHost = '', esUser = '', esPass = '') => { if (!esHost) { @@ -38,7 +30,7 @@ export const buildEsClient = (esHost = '', esUser = '', esPass = '') => { }; } - return new Client(esConfig); + return SearchClient(esConfig); }; export const buildEsClientViaEnv = () => { From dbcce45e805a4d1db6becacad6c5fe38c184ac8e Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Fri, 30 Jan 2026 13:24:47 -0500 Subject: [PATCH 2/7] Working Test Client setup --- modules/server/package.json | 3 +- modules/server/src/searchClient/index.ts | 15 +++- package-lock.json | 99 ++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/modules/server/package.json b/modules/server/package.json index 0b1ca95fd..d2b7c7430 100644 --- a/modules/server/package.json +++ b/modules/server/package.json @@ -28,10 +28,11 @@ "watch": "npm run clear:dist && npm run build -- --watch" }, "dependencies": { - "@graphql-tools/merge": "^9.0.4", "@elastic/elasticsearch": "^7.17.14", + "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "apollo-server": "^3.10.3", "apollo-server-core": "^3.10.3", diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts index a1cbe93c1..b36b703b0 100644 --- a/modules/server/src/searchClient/index.ts +++ b/modules/server/src/searchClient/index.ts @@ -1,7 +1,16 @@ -import { Client, type ClientOptions } from '@elastic/elasticsearch'; +import { Client as ElasticClient, type ClientOptions as ESClientOptions } from '@elastic/elasticsearch'; +import { Client as OpenSearchClient, type ClientOptions as OSClientOptions } from '@opensearch-project/opensearch'; -const SearchClient = (options: ClientOptions) => { - return new Client(options); +const SearchClient = (options: ESClientOptions | OSClientOptions) => { + const clientIsElasticSearch = false; + + if (clientIsElasticSearch) { + const clientOptions = options as ESClientOptions; + return new ElasticClient(clientOptions); + } else { + const clientOptions = options as OSClientOptions; + return new OpenSearchClient(clientOptions); + } }; export type SearchClientType = ReturnType; diff --git a/package-lock.json b/package-lock.json index 5d7515a49..796448ee9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -188,6 +188,7 @@ "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "apollo-server": "^3.10.3", "apollo-server-core": "^3.10.3", @@ -4423,6 +4424,50 @@ "node": ">=12.4.0" } }, + "node_modules/@opensearch-project/opensearch": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-3.5.1.tgz", + "integrity": "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA==", + "license": "Apache-2.0", + "dependencies": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "json11": "^2.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "engines": { + "node": ">=14", + "yarn": "^1.22.10" + } + }, + "node_modules/@opensearch-project/opensearch/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@opensearch-project/opensearch/node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@overture-stack/arranger-components": { "resolved": "modules/components", "link": true @@ -6876,6 +6921,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", + "license": "MIT" + }, "node_modules/axe-core": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz", @@ -17553,6 +17604,15 @@ "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", "dev": true }, + "node_modules/json11": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/json11/-/json11-2.0.2.tgz", + "integrity": "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==", + "license": "MIT", + "bin": { + "json11": "dist/cli.mjs" + } + }, "node_modules/json3": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz", @@ -32248,6 +32308,34 @@ "integrity": "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==", "dev": true }, + "@opensearch-project/opensearch": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-3.5.1.tgz", + "integrity": "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA==", + "requires": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "json11": "^2.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "dependencies": { + "debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "requires": { + "ms": "^2.1.3" + } + }, + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + } + } + }, "@overture-stack/arranger-components": { "version": "file:modules/components", "requires": { @@ -32360,6 +32448,7 @@ "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "@tsconfig/node22": "^22.0.0", "@types/convert-units": "^2.3.5", @@ -34254,6 +34343,11 @@ "possible-typed-array-names": "^1.0.0" } }, + "aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==" + }, "axe-core": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz", @@ -42119,6 +42213,11 @@ "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", "dev": true }, + "json11": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/json11/-/json11-2.0.2.tgz", + "integrity": "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==" + }, "json3": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz", From 65290463f0ef28d0f0b73f75067033b9861e1e2b Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Fri, 30 Jan 2026 13:41:02 -0500 Subject: [PATCH 3/7] Additional Type cleanup --- .../src/admin/schemas/AggsState/utils.ts | 10 +++--- .../src/admin/schemas/ColumnsState/utils.ts | 8 ++--- .../admin/schemas/ExtendedMapping/utils.ts | 4 +-- .../src/admin/schemas/IndexSchema/utils.ts | 34 +++++++++---------- .../src/admin/schemas/MatchboxState/utils.ts | 8 ++--- modules/server/src/gqlServer.ts | 2 +- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index 3199ae4c9..38f774d0b 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -28,7 +28,7 @@ export const getAggsSetState = async (args: I_AggsStateQueryInput): Promise => { const { projectId, graphqlField } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((entry) => entry.name === graphqlField); - return metaData.config['aggs-state']; + return metaData?.config['aggs-state']; }; export const saveAggsSetState = @@ -36,12 +36,12 @@ export const saveAggsSetState = async (args: I_SaveAggsStateMutationInput): Promise => { const { graphqlField, projectId, state } = args; const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); - const currentAggsState = currentMetadata.config['aggs-state']; + const currentAggsState = currentMetadata?.config['aggs-state']; const sortByNewOrder = sortBy((i: I_AggsState) => state.findIndex((_i) => _i.field === i.field)); const newAggsSetState: typeof currentAggsState = { timestamp: timestamp(), state: sortByNewOrder( - currentAggsState.state.map((item) => ({ + currentAggsState?.state.map((item) => ({ ...(state.find((_item) => _item.field === item.field) || item), type: item.type, })), @@ -51,8 +51,8 @@ export const saveAggsSetState = await updateProjectIndexMetadata(es)({ projectId, metaData: { - index: currentMetadata.index, - name: currentMetadata.name, + index: currentMetadata?.index, + name: currentMetadata?.name, config: { 'aggs-state': newAggsSetState, }, diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index ace1b4771..1752eeb02 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -20,7 +20,7 @@ export const getColumnSetState = async (args: I_ColumnStateQueryInput): Promise => { const { graphqlField, projectId } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); - return metaData.config['columns-state']; + return metaData?.config['columns-state']; }; export const createColumnSetState = @@ -52,7 +52,7 @@ export const saveColumnState = ...state, columns: sortByNewOrder( replaceBy( - currentIndexMetadata.config['columns-state'].state.columns, + currentIndexMetadata?.config['columns-state']?.state?.columns, state.columns, (oldCol, newCol) => oldCol.field === newCol.field, ), @@ -61,8 +61,8 @@ export const saveColumnState = await updateProjectIndexMetadata(es)({ projectId, metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, + index: currentIndexMetadata?.index, + name: currentIndexMetadata?.name, config: { 'columns-state': { timestamp: timestamp(), diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index 038ce9e3c..fc00bef13 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -125,8 +125,8 @@ export const saveExtendedMapping = await updateProjectIndexMetadata(es)({ projectId, metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, + index: currentIndexMetadata?.index, + name: currentIndexMetadata?.name, config: { extended: newExtendedMapping, }, diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index 4686c9585..cdeb978a8 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -75,11 +75,24 @@ export const getProjectMetadata = })), ); +export const getProjectIndex = + (es: SearchClientType) => + async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { + try { + const output = (await getProjectMetadata(es)(projectId)).find( + ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, + ); + return output; + } catch { + throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); + } + }; + export const createNewIndex = (es: SearchClientType) => async (args: INewIndexInput): Promise => { const { projectId, graphqlField, esIndex } = args; - const arrangerProject: {} = (await getArrangerProjects(es)).find((project) => project.id === projectId); + const arrangerProject = (await getArrangerProjects(es)).find((project) => project.id === projectId); if (arrangerProject) { const serializedGqlField = serializeToGqlField(graphqlField); @@ -126,9 +139,7 @@ export const createNewIndex = // because different metadata entities write to the same ES document, update operations need to be queued up to a single concurrency controlled task queue for each project. This factory creates a task queue manager for this purpose. const createProjectQueueManager = () => { - const queues: { - [projectId: string]: any; - } = {}; + const queues: Record = {}; return { getQueue: (projectId: string) => { if (!queues[projectId]) { @@ -142,7 +153,7 @@ const createProjectQueueManager = () => { // pretty bad, since we're just taking anything right now in run time, but at least graphQl will ensure `metaData` is typed in runtime const projectQueueManager = createProjectQueueManager(); export const updateProjectIndexMetadata = - (es: Client) => + (es: SearchClientType) => async ({ projectId, metaData, @@ -168,19 +179,6 @@ export const updateProjectIndexMetadata = }); }; -export const getProjectIndex = - (es: SearchClientType) => - async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { - try { - const output = (await getProjectMetadata(es)(projectId)).find( - ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, - ); - return output; - } catch { - throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); - } - }; - export const removeProjectIndex = (es: SearchClientType) => async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 44fca9878..22a5018a0 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -30,7 +30,7 @@ export const getMatchBoxState = (es: SearchClientType) => async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); - return currentMetadata.config['matchbox-state']; + return currentMetadata?.config['matchbox-state']; }; export const saveMatchBoxState = @@ -41,7 +41,7 @@ export const saveMatchBoxState = state: updatedMatchboxFields, }: I_SaveMatchBoxStateMutationInput): Promise => { const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); - const currentMatchboxFields = currentMetadata.config['matchbox-state'].state; + const currentMatchboxFields = currentMetadata?.config['matchbox-state'].state || []; const newMatchboxState: I_MatchBoxState = { timestamp: timestamp(), state: replaceBy( @@ -54,8 +54,8 @@ export const saveMatchBoxState = await updateProjectIndexMetadata(es)({ projectId, metaData: { - index: currentMetadata.index, - name: currentMetadata.name, + index: currentMetadata?.index, + name: currentMetadata?.name, config: { 'matchbox-state': newMatchboxState, }, diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index 5e4ec62e9..38e1ac2ec 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -20,7 +20,7 @@ export type ResolverOutput = T | Promise; * @return Returns resolved value; */ type DefaultRoot = Root; -export type Resolver = ( +export type Resolver = ( root: Root, args: QueryArgs, context: Context, From ca8179a99570e956f7ca27029d911182f43842fe Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Fri, 30 Jan 2026 14:25:59 -0500 Subject: [PATCH 4/7] Basic check for version.distribution, remove console.log --- modules/server/src/download/index.js | 2 -- modules/server/src/searchClient/index.ts | 13 +++++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/server/src/download/index.js b/modules/server/src/download/index.js index e92b659ae..42b8ea591 100644 --- a/modules/server/src/download/index.js +++ b/modules/server/src/download/index.js @@ -91,8 +91,6 @@ export const dataStream = async ({ ctx, params }) => { }; const download = ({ enableAdmin }) => { - console.log('enableAdmin', enableAdmin); - const router = Router(); router.use(urlencoded({ extended: true })); diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts index b36b703b0..6517e8582 100644 --- a/modules/server/src/searchClient/index.ts +++ b/modules/server/src/searchClient/index.ts @@ -1,16 +1,21 @@ import { Client as ElasticClient, type ClientOptions as ESClientOptions } from '@elastic/elasticsearch'; import { Client as OpenSearchClient, type ClientOptions as OSClientOptions } from '@opensearch-project/opensearch'; -const SearchClient = (options: ESClientOptions | OSClientOptions) => { - const clientIsElasticSearch = false; +import { ENV_CONFIG } from '#config/index.js'; - if (clientIsElasticSearch) { +const SearchClient = async (options: ESClientOptions | OSClientOptions) => { + const { ES_HOST } = ENV_CONFIG; + const searchConfig = await (await fetch(ES_HOST)).json(); + const { distribution } = searchConfig.version; + + if (distribution === 'elasticsearch') { const clientOptions = options as ESClientOptions; return new ElasticClient(clientOptions); - } else { + } else if (distribution === 'opensearch') { const clientOptions = options as OSClientOptions; return new OpenSearchClient(clientOptions); } + throw new Error(`Error configuring Search Client with distribution ${distribution}`); }; export type SearchClientType = ReturnType; From 1acb1ef13d0d05826bc819a3571e9dee7a8b2880 Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Mon, 9 Feb 2026 19:46:07 -0500 Subject: [PATCH 5/7] Updated Client Init, with corrected use of Promises & Types --- modules/server/src/admin/index.ts | 38 ++++++------ .../src/admin/schemas/AggsState/index.ts | 15 ++--- .../src/admin/schemas/AggsState/resolvers.ts | 51 ++++++++-------- .../src/admin/schemas/AggsState/utils.ts | 8 +-- .../src/admin/schemas/ColumnsState/utils.ts | 8 +-- .../admin/schemas/ExtendedMapping/utils.ts | 10 +-- .../src/admin/schemas/IndexSchema/utils.ts | 16 ++--- .../src/admin/schemas/MatchboxState/utils.ts | 6 +- .../src/admin/schemas/ProjectSchema/index.ts | 16 ++--- .../admin/schemas/ProjectSchema/resolvers.ts | 61 ++++++++----------- .../src/admin/schemas/ProjectSchema/utils.ts | 8 +-- modules/server/src/admin/schemas/types.ts | 25 ++++---- .../src/admin/services/elasticsearch/index.ts | 8 +-- modules/server/src/admin/types.ts | 4 +- modules/server/src/config/utils/index.ts | 4 +- modules/server/src/gqlServer.ts | 4 +- modules/server/src/mapping/utils/esSearch.ts | 6 +- .../server/src/mapping/utils/fetchMapping.ts | 8 +-- modules/server/src/schema/Root.ts | 4 +- modules/server/src/searchClient/index.ts | 32 ++++++---- modules/server/src/server.ts | 12 ++-- 21 files changed, 173 insertions(+), 171 deletions(-) diff --git a/modules/server/src/admin/index.ts b/modules/server/src/admin/index.ts index 4b1b06df0..1b3aceacc 100644 --- a/modules/server/src/admin/index.ts +++ b/modules/server/src/admin/index.ts @@ -5,7 +5,7 @@ import { ApolloServer } from 'apollo-server-express'; import { type GraphQLSchema } from 'graphql'; import { print } from 'graphql/language/printer'; -import { type SearchClientType } from '../searchClient/index.js'; +import { type AllClients } from '../searchClient/index.js'; import { createAggsStateByIndexResolver, @@ -67,30 +67,30 @@ function buildElasticsearchClient(config: AdminApiConfig) { return createElasticsearchClient(config.esHost, config.esUser, config.esPass); } -const initialize = (config: AdminApiConfig): Promise => - new Promise(async (resolve, reject) => { - console.info('Initializing Elasticsearch Client for host: ' + config.esHost); - const esClient = buildElasticsearchClient(config); - try { - console.info('Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch'); - const exists = await esClient.indices.exists({ +const initialize = async (config: AdminApiConfig): Promise => { + console.info('Initializing Elasticsearch Client for host: ' + config.esHost); + const esClient = await buildElasticsearchClient(config); + try { + console.info('Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch'); + const exists = await esClient.indices.exists({ + index: constants.ARRANGER_PROJECT_INDEX, + }); + if (!exists) { + esClient.indices.create({ index: constants.ARRANGER_PROJECT_INDEX, }); - if (!exists) { - esClient.indices.create({ - index: constants.ARRANGER_PROJECT_INDEX, - }); - } - resolve(esClient); - } catch (err) { - setTimeout(() => { - initialize(config).then(() => resolve(esClient)); - }, 1000); } - }); + return esClient; + } catch (err) { + setTimeout(async () => { + return await initialize(config); + }, 1000); + } +}; export default async (config: AdminApiConfig) => { const esClient = await initialize(config); + if (!esClient) throw new Error('Could not initialize esClient'); return new ApolloServer({ schema: await createSchema(), context: (): IQueryContext => ({ diff --git a/modules/server/src/admin/schemas/AggsState/index.ts b/modules/server/src/admin/schemas/AggsState/index.ts index c82969520..5679f87bc 100644 --- a/modules/server/src/admin/schemas/AggsState/index.ts +++ b/modules/server/src/admin/schemas/AggsState/index.ts @@ -1,11 +1,12 @@ -import resolvers from './resolvers'; -import typeDefs from './schemaTypeDefs'; import { makeExecutableSchema } from 'graphql-tools'; +import resolvers from './resolvers.js'; +import typeDefs from './schemaTypeDefs.js'; + export const createSchema = async () => { - const schema = makeExecutableSchema({ - typeDefs: await typeDefs(), - resolvers, - }); - return schema; + const schema = makeExecutableSchema({ + typeDefs: await typeDefs(), + resolvers, + }); + return schema; }; diff --git a/modules/server/src/admin/schemas/AggsState/resolvers.ts b/modules/server/src/admin/schemas/AggsState/resolvers.ts index 397d19c6c..21138942e 100644 --- a/modules/server/src/admin/schemas/AggsState/resolvers.ts +++ b/modules/server/src/admin/schemas/AggsState/resolvers.ts @@ -1,33 +1,34 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { IQueryContext } from '../../types'; -import { I_AggsSetState, I_AggsStateQueryInput, I_SaveAggsStateMutationInput } from './types'; -import { getAggsSetState, saveAggsSetState } from './utils'; -import { Resolver } from '../types'; +import { type GraphQLResolveInfo } from 'graphql'; -const saveAggsStateMutationResolver: Resolver = - async ( - obj: {}, - args, - { es }: IQueryContext, - info: GraphQLResolveInfo, - ): Promise => { - return await saveAggsSetState(es)(args); - }; +import { type IQueryContext } from '../../types.js'; +import { type Resolver } from '../types.js'; + +import { type I_AggsSetState, type I_AggsStateQueryInput, type I_SaveAggsStateMutationInput } from './types.js'; +import { getAggsSetState, saveAggsSetState } from './utils.js'; + +const saveAggsStateMutationResolver: Resolver = async ( + obj: object, + args, + { es }: IQueryContext, + info: GraphQLResolveInfo, +): Promise => { + return await saveAggsSetState(es)(args); +}; const aggsStateQueryResolver: Resolver = ( - obj: {}, - args, - { es }: IQueryContext, - info: GraphQLResolveInfo, + obj: object, + args, + { es }: IQueryContext, + info: GraphQLResolveInfo, ): Promise => { - return getAggsSetState(es)(args); + return getAggsSetState(es)(args); }; export default { - Query: { - aggsState: aggsStateQueryResolver, - }, - Mutation: { - saveAggsState: saveAggsStateMutationResolver, - }, + Query: { + aggsState: aggsStateQueryResolver, + }, + Mutation: { + saveAggsState: saveAggsStateMutationResolver, + }, }; diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index 38f774d0b..7727bda38 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -1,7 +1,7 @@ import { sortBy } from 'ramda'; import { mappingToAggsState } from '../../../mapping/index.js'; -import { type SearchClientType } from '../../../searchClient/index.js'; +import { type AllClients } from '../../../searchClient/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; import { timestamp } from '../../services/index.js'; import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; @@ -15,7 +15,7 @@ import { } from './types.js'; export const createAggsSetState = - (es: SearchClientType) => + (es: AllClients) => async ({ esIndex }: EsIndexLocation): Promise => { const rawEsmapping = await getEsMapping(es)({ esIndex }); const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; @@ -24,7 +24,7 @@ export const createAggsSetState = }; export const getAggsSetState = - (es: SearchClientType) => + (es: AllClients) => async (args: I_AggsStateQueryInput): Promise => { const { projectId, graphqlField } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((entry) => entry.name === graphqlField); @@ -32,7 +32,7 @@ export const getAggsSetState = }; export const saveAggsSetState = - (es: SearchClientType) => + (es: AllClients) => async (args: I_SaveAggsStateMutationInput): Promise => { const { graphqlField, projectId, state } = args; const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index 1752eeb02..f433e2202 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -1,6 +1,6 @@ import { sortBy } from 'ramda'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { mappingToColumnsState } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -16,7 +16,7 @@ import { } from './types.js'; export const getColumnSetState = - (es: SearchClientType) => + (es: AllClients) => async (args: I_ColumnStateQueryInput): Promise => { const { graphqlField, projectId } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); @@ -24,7 +24,7 @@ export const getColumnSetState = }; export const createColumnSetState = - (es: SearchClientType) => + (es: AllClients) => async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { const rawEsmapping = await getEsMapping(es)({ esIndex, @@ -43,7 +43,7 @@ export const createColumnSetState = }; export const saveColumnState = - (es: SearchClientType) => + (es: AllClients) => async ({ graphqlField, projectId, state }: I_SaveColumnsStateMutationInput): Promise => { const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index fc00bef13..bc0ba673c 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -1,6 +1,6 @@ import { UserInputError } from 'apollo-server'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { extendMapping } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -16,7 +16,7 @@ import { } from './types.js'; export const createExtendedMapping = - (es: SearchClientType) => + (es: AllClients) => async ({ esIndex }: EsIndexLocation): Promise => { let extendedMappings: I_GqlExtendedFieldMapping[] = []; try { @@ -32,7 +32,7 @@ export const createExtendedMapping = }; export const getExtendedMapping = - (es: SearchClientType) => + (es: AllClients) => async ({ projectId, graphqlField, @@ -66,7 +66,7 @@ export const getExtendedMapping = }; export const updateFieldExtendedMapping = - (es: SearchClientType) => + (es: AllClients) => async ({ field: mutatedField, graphqlField, @@ -106,7 +106,7 @@ export const updateFieldExtendedMapping = }; export const saveExtendedMapping = - (es: SearchClientType) => + (es: AllClients) => async (args: I_SaveExtendedMappingMutationArgs): Promise => { const { projectId, graphqlField, input } = args; const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index cdeb978a8..648b93bd5 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -1,7 +1,7 @@ import { UserInputError } from 'apollo-server'; import Qew from 'qew'; // TODO: using 0.9.13 because later versions break the async -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { constants } from '../../services/constants.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -33,7 +33,7 @@ export const getProjectMetadataEsLocation = ( }); const mappingExistsOn = - (es: SearchClientType) => + (es: AllClients) => async ({ esIndex }: EsIndexLocation): Promise => { try { await getEsMapping(es)({ esIndex }); @@ -44,7 +44,7 @@ const mappingExistsOn = }; export const getProjectStorageMetadata = - (es: SearchClientType) => + (es: AllClients) => async (projectId: string): Promise => { try { const { @@ -61,7 +61,7 @@ export const getProjectStorageMetadata = }; export const getProjectMetadata = - (es: SearchClientType) => + (es: AllClients) => async (projectId: string): Promise => Promise.all( (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ @@ -76,7 +76,7 @@ export const getProjectMetadata = ); export const getProjectIndex = - (es: SearchClientType) => + (es: AllClients) => async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { try { const output = (await getProjectMetadata(es)(projectId)).find( @@ -89,7 +89,7 @@ export const getProjectIndex = }; export const createNewIndex = - (es: SearchClientType) => + (es: AllClients) => async (args: INewIndexInput): Promise => { const { projectId, graphqlField, esIndex } = args; const arrangerProject = (await getArrangerProjects(es)).find((project) => project.id === projectId); @@ -153,7 +153,7 @@ const createProjectQueueManager = () => { // pretty bad, since we're just taking anything right now in run time, but at least graphQl will ensure `metaData` is typed in runtime const projectQueueManager = createProjectQueueManager(); export const updateProjectIndexMetadata = - (es: SearchClientType) => + (es: AllClients) => async ({ projectId, metaData, @@ -180,7 +180,7 @@ export const updateProjectIndexMetadata = }; export const removeProjectIndex = - (es: SearchClientType) => + (es: AllClients) => async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { try { const removedIndexMetadata = await getProjectIndex(es)({ diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 22a5018a0..1deaee0ed 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -1,4 +1,4 @@ -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping/index.js'; import { replaceBy, timestamp } from '../../services/index.js'; @@ -27,14 +27,14 @@ export const createMatchboxState = ({ }; export const getMatchBoxState = - (es: SearchClientType) => + (es: AllClients) => async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); return currentMetadata?.config['matchbox-state']; }; export const saveMatchBoxState = - (es: SearchClientType) => + (es: AllClients) => async ({ graphqlField, projectId, diff --git a/modules/server/src/admin/schemas/ProjectSchema/index.ts b/modules/server/src/admin/schemas/ProjectSchema/index.ts index 489cbcc60..71b3654f3 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/index.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/index.ts @@ -1,14 +1,14 @@ import { addMocksToSchema } from '@graphql-tools/mock'; import { makeExecutableSchema } from 'graphql-tools'; -import { createResolvers } from './resolvers'; -import typeDefs from './schemaTypeDefs'; +import { createResolvers } from './resolvers.js'; +import typeDefs from './schemaTypeDefs.js'; export const createSchema = async () => { - const schema = makeExecutableSchema({ - typeDefs: await typeDefs(), - resolvers: await createResolvers(), - }); - addMocksToSchema({ schema, preserveResolvers: true }); - return schema; + const schema = makeExecutableSchema({ + typeDefs: await typeDefs(), + resolvers: await createResolvers(), + }); + addMocksToSchema({ schema, preserveResolvers: true }); + return schema; }; diff --git a/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts b/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts index 711ccccfe..769ed13eb 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts @@ -1,45 +1,36 @@ -import { addArrangerProject, getArrangerProjects, removeArrangerProject } from './utils'; -import { IArrangerProject, IProjectQueryInput } from './types'; -import { Resolver } from '../types'; +import { type Resolver } from '../types.js'; -const projectsQueryResolver: Resolver> = async (_, args, { es }, info) => - getArrangerProjects(es); +import { type IArrangerProject, type IProjectQueryInput } from './types.js'; +import { addArrangerProject, getArrangerProjects, removeArrangerProject } from './utils.js'; -const singleProjectQueryResolver: Resolver = async ( - _, - { id }, - { es }, - info, -) => { - const projects = await getArrangerProjects(es); - return projects.find(({ id: _id }: { id: String }) => id === _id); +const projectsQueryResolver: Resolver = async (_, args, { es }, info) => + await getArrangerProjects(es); + +const singleProjectQueryResolver: Resolver = async (_, { id }, { es }, info) => { + const projects = await getArrangerProjects(es); + return projects.find(({ id: _id }: { id: string }) => id === _id); }; -const newProjectMutationResolver: Resolver = async ( - _, - { id }, - { es }, - info, -) => - addArrangerProject(es)(id).catch((err: Error) => { - err.message = 'potential project ID conflict'; - return Promise.reject(err); - }); +const newProjectMutationResolver: Resolver = async (_, { id }, { es }, info) => + addArrangerProject(es)(id).catch((err: Error) => { + err.message = 'potential project ID conflict'; + return Promise.reject(err); + }); const deleteProjectMutationResolver: Resolver = async ( - _, - { id }, - { es }, - info, + _, + { id }, + { es }, + info, ) => removeArrangerProject(es)(id); export const createResolvers = async () => ({ - Query: { - projects: projectsQueryResolver, - project: singleProjectQueryResolver, - }, - Mutation: { - newProject: newProjectMutationResolver, - deleteProject: deleteProjectMutationResolver, - }, + Query: { + projects: projectsQueryResolver, + project: singleProjectQueryResolver, + }, + Mutation: { + newProject: newProjectMutationResolver, + deleteProject: deleteProjectMutationResolver, + }, }); diff --git a/modules/server/src/admin/schemas/ProjectSchema/utils.ts b/modules/server/src/admin/schemas/ProjectSchema/utils.ts index 7ba000041..e17d61876 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/utils.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/utils.ts @@ -1,4 +1,4 @@ -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { constants } from '../../services/constants.js'; import { serializeToEsId } from '../../services/index.js'; @@ -14,7 +14,7 @@ export const newArrangerProject = (id: string): IArrangerProject => ({ timestamp: new Date().toISOString(), }); -export const getArrangerProjects = async (es: SearchClientType): Promise => { +export const getArrangerProjects = async (es: AllClients): Promise => { const { body: { hits: { hits }, @@ -42,7 +42,7 @@ export const getArrangerProjects = async (es: SearchClientType): Promise + (es: AllClients) => async (id: string): Promise => { //id must be lower case const _id = serializeToEsId(id); @@ -63,7 +63,7 @@ export const addArrangerProject = }; export const removeArrangerProject = - (es: SearchClientType) => + (es: AllClients) => async (id: string): Promise => { const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); if (existingProject) { diff --git a/modules/server/src/admin/schemas/types.ts b/modules/server/src/admin/schemas/types.ts index 31cb178aa..5366fea92 100644 --- a/modules/server/src/admin/schemas/types.ts +++ b/modules/server/src/admin/schemas/types.ts @@ -1,18 +1,19 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { IQueryContext } from '../types'; -import { MergeInfo } from 'graphql-tools'; +import { type GraphQLResolveInfo } from 'graphql'; +import { type MergeInfo } from 'graphql-tools'; + +import { type IQueryContext } from '../types.js'; export type ResolverOutput = T | Promise; export interface EsIndexLocation { - esIndex: string; + esIndex: string; } -export type Resolver = - | (( - a: any, - args: Args, - c: IQueryContext, - d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, - ) => ResolverOutput) - | ResolverOutput; +export type Resolver = + | (( + a: any, + args: Args, + c: IQueryContext, + d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, + ) => ResolverOutput) + | ResolverOutput; diff --git a/modules/server/src/admin/services/elasticsearch/index.ts b/modules/server/src/admin/services/elasticsearch/index.ts index 90f3a4206..a67bad81e 100644 --- a/modules/server/src/admin/services/elasticsearch/index.ts +++ b/modules/server/src/admin/services/elasticsearch/index.ts @@ -1,8 +1,8 @@ -import SearchClient, { type SearchClientType } from '#searchClient/index.js'; +import getSearchClient, { type AllClients } from '#searchClient/index.js'; import { type EsMapping } from './types.js'; -export const createClient = (esHost: string, esUser: string, esPass: string) => { +export const createClient = async (esHost: string, esUser: string, esPass: string) => { const esConf = { node: esHost }; if (esUser && esPass) { esConf['auth'] = { @@ -10,11 +10,11 @@ export const createClient = (esHost: string, esUser: string, esPass: string) => password: esPass, }; } - return SearchClient(esConf); + return await getSearchClient(esConf); }; export const getEsMapping = - (es: SearchClientType) => + (es: AllClients) => async ({ esIndex, }: { diff --git a/modules/server/src/admin/types.ts b/modules/server/src/admin/types.ts index ce5c6d497..5a04b4e23 100644 --- a/modules/server/src/admin/types.ts +++ b/modules/server/src/admin/types.ts @@ -1,7 +1,7 @@ import { type GraphQLResolveInfo } from 'graphql'; import { type MergeInfo } from 'graphql-tools'; -import { type SearchClientType } from '../searchClient/index.js'; +import { type AllClients } from '../searchClient/index.js'; export type AdminApiConfig = { esHost: string; @@ -9,7 +9,7 @@ export type AdminApiConfig = { esPass: string; }; export type IQueryContext = { - es: SearchClientType; + es: AllClients; }; export type ResolverOutput = T | Promise; diff --git a/modules/server/src/config/utils/index.ts b/modules/server/src/config/utils/index.ts index 6166bdea7..48b7c4ca0 100644 --- a/modules/server/src/config/utils/index.ts +++ b/modules/server/src/config/utils/index.ts @@ -1,13 +1,13 @@ import { ENV_CONFIG } from '#config/index.js'; import { type ConfigObject, configProperties } from '#config/types.js'; import { setsMapping } from '#schema/index.js'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; export const initializeSets = async ({ esClient, setsIndex: setsIndexParam, }: { - esClient: SearchClientType; + esClient: AllClients; setsIndex: string; }): Promise => { ENV_CONFIG.DEBUG_MODE && console.log(`Attempting to create Sets index "${setsIndexParam}"...`); diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index 38e1ac2ec..af8908d03 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -1,9 +1,9 @@ import { type GraphQLResolveInfo } from 'graphql'; -import { type SearchClientType } from './searchClient/index.js'; +import { type AllClients } from './searchClient/index.js'; export type Context = { - esClient: SearchClientType; + esClient: AllClients; }; export type Root = Record; diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts index 82b4f16e3..46cd11918 100644 --- a/modules/server/src/mapping/utils/esSearch.ts +++ b/modules/server/src/mapping/utils/esSearch.ts @@ -1,7 +1,7 @@ import { type RequestParams } from '@elastic/elasticsearch'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; -export default (esClient: SearchClientType) => async (params: RequestParams.Search) => { - return (await esClient?.search(params))?.body; +export default (esClient: AllClients) => async (params: RequestParams.Search) => { + return esClient.search(params)?.body; }; diff --git a/modules/server/src/mapping/utils/fetchMapping.ts b/modules/server/src/mapping/utils/fetchMapping.ts index a6dce4f8f..91d837ffa 100644 --- a/modules/server/src/mapping/utils/fetchMapping.ts +++ b/modules/server/src/mapping/utils/fetchMapping.ts @@ -1,8 +1,8 @@ import type { CatAliasesAliasesRecord } from '@elastic/elasticsearch/api/types'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; -export const getESAliases = async (esClient: SearchClientType) => { +export const getESAliases = async (esClient: AllClients) => { const { body } = await esClient.cat.aliases({ format: 'json' }); return body; @@ -11,7 +11,7 @@ export const getESAliases = async (esClient: SearchClientType) => { export const checkESAlias = (aliases: CatAliasesAliasesRecord[], possibleAlias: string) => aliases?.find((foundIndex = { alias: undefined }) => foundIndex.alias === possibleAlias)?.index; -export const fetchMapping = async ({ esClient, index }: { esClient: SearchClientType; index: string }) => { +export const fetchMapping = async ({ esClient, index }: { esClient: AllClients; index: string }) => { if (esClient) { console.log(`Fetching ES mapping for "${index}"...`); const aliases = await getESAliases(esClient); @@ -20,7 +20,7 @@ export const fetchMapping = async ({ esClient, index }: { esClient: SearchClient const accessor = alias || index; - const mapping = await esClient?.indices + const mapping = await esClient.indices .getMapping({ index: accessor, }) diff --git a/modules/server/src/schema/Root.ts b/modules/server/src/schema/Root.ts index 363f675ba..a742581d4 100644 --- a/modules/server/src/schema/Root.ts +++ b/modules/server/src/schema/Root.ts @@ -6,7 +6,7 @@ import Parallel from 'paralleljs'; import { ENV_CONFIG } from '#config/index.js'; import { createConnectionResolvers, saveSet, mappingToFields } from '#mapping/index.js'; import { checkESAlias, getESAliases } from '#mapping/utils/fetchMapping.js'; -import { type SearchClientType } from '#searchClient/index.js'; +import { type AllClients } from '#searchClient/index.js'; import { typeDefs as AggregationsTypeDefs } from './Aggregations.js'; import ConfigsTypeDefs from './configQuery.js'; @@ -82,7 +82,7 @@ export const resolvers = ({ enableAdmin, types, rootTypes, scalarTypes, getServe Date: GraphQLDate, Root: { viewer: resolveObject, - hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: SearchClientType }) => { + hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: AllClients }) => { if (documentType) { if (index) { const [_, type] = types.find(([name]) => name === documentType) || []; diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts index 6517e8582..3980ff7b3 100644 --- a/modules/server/src/searchClient/index.ts +++ b/modules/server/src/searchClient/index.ts @@ -3,21 +3,29 @@ import { Client as OpenSearchClient, type ClientOptions as OSClientOptions } fro import { ENV_CONFIG } from '#config/index.js'; -const SearchClient = async (options: ESClientOptions | OSClientOptions) => { +export type AllClients = ElasticClient | OpenSearchClient; +type Clients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; +type ClientTypes = keyof Clients; +type ClientOptions = { elasticsearch: ESClientOptions; opensearch: OSClientOptions }; + +async function getSearchClient(options: ClientOptions[Key]) { const { ES_HOST } = ENV_CONFIG; const searchConfig = await (await fetch(ES_HOST)).json(); const { distribution } = searchConfig.version; - - if (distribution === 'elasticsearch') { - const clientOptions = options as ESClientOptions; - return new ElasticClient(clientOptions); - } else if (distribution === 'opensearch') { - const clientOptions = options as OSClientOptions; - return new OpenSearchClient(clientOptions); + try { + if (distribution === 'opensearch') { + const clientOptions = options as OSClientOptions; + return new OpenSearchClient(clientOptions); + } else { + const clientOptions = options as ESClientOptions; + return new ElasticClient(clientOptions); + } + } catch (error) { + console.error(error); + throw new Error(`Error configuring Search Client`); } - throw new Error(`Error configuring Search Client with distribution ${distribution}`); -}; +} -export type SearchClientType = ReturnType; +export type SearchClientType = ReturnType; -export default SearchClient; +export default getSearchClient; diff --git a/modules/server/src/server.ts b/modules/server/src/server.ts index 035ff9d99..83bdc9914 100644 --- a/modules/server/src/server.ts +++ b/modules/server/src/server.ts @@ -6,12 +6,12 @@ import { ENABLE_LOGS, ES_ARRANGER_SET_INDEX, ENABLE_NETWORK_AGGREGATION } from ' import { ENV_CONFIG } from './config/index.js'; import downloadRoutes from './download/index.js'; import getGraphQLRoutes from './graphqlRoutes.js'; -import SearchClient from './searchClient/index.js'; +import getSearchClient from './searchClient/index.js'; import getDefaultServerSideFilter from './utils/getDefaultServerSideFilter.js'; const { CONFIG_FILES_PATH, DEBUG_MODE, ENABLE_ADMIN, ES_HOST, ES_USER, ES_PASS, PING_PATH } = ENV_CONFIG; -export const buildEsClient = (esHost = '', esUser = '', esPass = '') => { +export const buildEsClient = async (esHost = '', esUser = '', esPass = '') => { if (!esHost) { console.error('no elasticsearch host was provided'); } @@ -30,11 +30,11 @@ export const buildEsClient = (esHost = '', esUser = '', esPass = '') => { }; } - return SearchClient(esConfig); + return await getSearchClient(esConfig); }; -export const buildEsClientViaEnv = () => { - return buildEsClient(ES_HOST, ES_USER, ES_PASS); +export const buildEsClientViaEnv = async () => { + return await buildEsClient(ES_HOST, ES_USER, ES_PASS); }; const arrangerServer = async ({ @@ -51,7 +51,7 @@ const arrangerServer = async ({ pingPath = PING_PATH, setsIndex = ES_ARRANGER_SET_INDEX, } = {}): Promise => { - const esClient = customEsClient || buildEsClient(esHost, esUser, esPass); + const esClient = customEsClient || (await buildEsClient(esHost, esUser, esPass)); const router = Router(); console.log('------------------------------------'); From a7747216ff2b048049bd86f46fccf4bd63d93493 Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Wed, 11 Feb 2026 14:43:18 -0500 Subject: [PATCH 6/7] Refine Types and Function definitions --- modules/server/.env.schema | 3 +- modules/server/src/admin/index.ts | 4 +- .../src/admin/schemas/AggsState/utils.ts | 8 ++-- .../src/admin/schemas/ColumnsState/utils.ts | 8 ++-- .../admin/schemas/ExtendedMapping/utils.ts | 10 ++--- .../src/admin/schemas/IndexSchema/utils.ts | 16 ++++---- .../src/admin/schemas/MatchboxState/utils.ts | 6 +-- .../src/admin/schemas/ProjectSchema/utils.ts | 8 ++-- .../src/admin/services/elasticsearch/index.ts | 4 +- modules/server/src/admin/types.ts | 4 +- modules/server/src/config/constants.ts | 1 + modules/server/src/config/utils/index.ts | 4 +- modules/server/src/gqlServer.ts | 4 +- modules/server/src/mapping/utils/esSearch.ts | 4 +- .../server/src/mapping/utils/fetchMapping.ts | 6 +-- modules/server/src/schema/Root.ts | 4 +- modules/server/src/searchClient/index.ts | 37 +++++++++++++------ 17 files changed, 73 insertions(+), 58 deletions(-) diff --git a/modules/server/.env.schema b/modules/server/.env.schema index a4bd01f09..8b63dd6ae 100644 --- a/modules/server/.env.schema +++ b/modules/server/.env.schema @@ -17,4 +17,5 @@ MAX_RESULTS_WINDOW=10000 PING_MS=2200 PING_PATH=/ping PORT=5050 -ROW_ID_FIELD_NAME=id \ No newline at end of file +ROW_ID_FIELD_NAME=id +SEARCH_CLIENT='' \ No newline at end of file diff --git a/modules/server/src/admin/index.ts b/modules/server/src/admin/index.ts index 1b3aceacc..be50a5d76 100644 --- a/modules/server/src/admin/index.ts +++ b/modules/server/src/admin/index.ts @@ -5,7 +5,7 @@ import { ApolloServer } from 'apollo-server-express'; import { type GraphQLSchema } from 'graphql'; import { print } from 'graphql/language/printer'; -import { type AllClients } from '../searchClient/index.js'; +import { type SearchClient } from '../searchClient/index.js'; import { createAggsStateByIndexResolver, @@ -67,7 +67,7 @@ function buildElasticsearchClient(config: AdminApiConfig) { return createElasticsearchClient(config.esHost, config.esUser, config.esPass); } -const initialize = async (config: AdminApiConfig): Promise => { +const initialize = async (config: AdminApiConfig): Promise => { console.info('Initializing Elasticsearch Client for host: ' + config.esHost); const esClient = await buildElasticsearchClient(config); try { diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index 7727bda38..e034cae4c 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -1,7 +1,7 @@ import { sortBy } from 'ramda'; import { mappingToAggsState } from '../../../mapping/index.js'; -import { type AllClients } from '../../../searchClient/index.js'; +import { type SearchClient } from '../../../searchClient/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; import { timestamp } from '../../services/index.js'; import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; @@ -15,7 +15,7 @@ import { } from './types.js'; export const createAggsSetState = - (es: AllClients) => + (es: SearchClient) => async ({ esIndex }: EsIndexLocation): Promise => { const rawEsmapping = await getEsMapping(es)({ esIndex }); const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; @@ -24,7 +24,7 @@ export const createAggsSetState = }; export const getAggsSetState = - (es: AllClients) => + (es: SearchClient) => async (args: I_AggsStateQueryInput): Promise => { const { projectId, graphqlField } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((entry) => entry.name === graphqlField); @@ -32,7 +32,7 @@ export const getAggsSetState = }; export const saveAggsSetState = - (es: AllClients) => + (es: SearchClient) => async (args: I_SaveAggsStateMutationInput): Promise => { const { graphqlField, projectId, state } = args; const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index f433e2202..e644cd181 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -1,6 +1,6 @@ import { sortBy } from 'ramda'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { mappingToColumnsState } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -16,7 +16,7 @@ import { } from './types.js'; export const getColumnSetState = - (es: AllClients) => + (es: SearchClient) => async (args: I_ColumnStateQueryInput): Promise => { const { graphqlField, projectId } = args; const metaData = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); @@ -24,7 +24,7 @@ export const getColumnSetState = }; export const createColumnSetState = - (es: AllClients) => + (es: SearchClient) => async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { const rawEsmapping = await getEsMapping(es)({ esIndex, @@ -43,7 +43,7 @@ export const createColumnSetState = }; export const saveColumnState = - (es: AllClients) => + (es: SearchClient) => async ({ graphqlField, projectId, state }: I_SaveColumnsStateMutationInput): Promise => { const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index bc0ba673c..15a463a31 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -1,6 +1,6 @@ import { UserInputError } from 'apollo-server'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { extendMapping } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -16,7 +16,7 @@ import { } from './types.js'; export const createExtendedMapping = - (es: AllClients) => + (es: SearchClient) => async ({ esIndex }: EsIndexLocation): Promise => { let extendedMappings: I_GqlExtendedFieldMapping[] = []; try { @@ -32,7 +32,7 @@ export const createExtendedMapping = }; export const getExtendedMapping = - (es: AllClients) => + (es: SearchClient) => async ({ projectId, graphqlField, @@ -66,7 +66,7 @@ export const getExtendedMapping = }; export const updateFieldExtendedMapping = - (es: AllClients) => + (es: SearchClient) => async ({ field: mutatedField, graphqlField, @@ -106,7 +106,7 @@ export const updateFieldExtendedMapping = }; export const saveExtendedMapping = - (es: AllClients) => + (es: SearchClient) => async (args: I_SaveExtendedMappingMutationArgs): Promise => { const { projectId, graphqlField, input } = args; const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index 648b93bd5..df061120d 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -1,7 +1,7 @@ import { UserInputError } from 'apollo-server'; import Qew from 'qew'; // TODO: using 0.9.13 because later versions break the async -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { constants } from '../../services/constants.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; @@ -33,7 +33,7 @@ export const getProjectMetadataEsLocation = ( }); const mappingExistsOn = - (es: AllClients) => + (es: SearchClient) => async ({ esIndex }: EsIndexLocation): Promise => { try { await getEsMapping(es)({ esIndex }); @@ -44,7 +44,7 @@ const mappingExistsOn = }; export const getProjectStorageMetadata = - (es: AllClients) => + (es: SearchClient) => async (projectId: string): Promise => { try { const { @@ -61,7 +61,7 @@ export const getProjectStorageMetadata = }; export const getProjectMetadata = - (es: AllClients) => + (es: SearchClient) => async (projectId: string): Promise => Promise.all( (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ @@ -76,7 +76,7 @@ export const getProjectMetadata = ); export const getProjectIndex = - (es: AllClients) => + (es: SearchClient) => async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { try { const output = (await getProjectMetadata(es)(projectId)).find( @@ -89,7 +89,7 @@ export const getProjectIndex = }; export const createNewIndex = - (es: AllClients) => + (es: SearchClient) => async (args: INewIndexInput): Promise => { const { projectId, graphqlField, esIndex } = args; const arrangerProject = (await getArrangerProjects(es)).find((project) => project.id === projectId); @@ -153,7 +153,7 @@ const createProjectQueueManager = () => { // pretty bad, since we're just taking anything right now in run time, but at least graphQl will ensure `metaData` is typed in runtime const projectQueueManager = createProjectQueueManager(); export const updateProjectIndexMetadata = - (es: AllClients) => + (es: SearchClient) => async ({ projectId, metaData, @@ -180,7 +180,7 @@ export const updateProjectIndexMetadata = }; export const removeProjectIndex = - (es: AllClients) => + (es: SearchClient) => async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { try { const removedIndexMetadata = await getProjectIndex(es)({ diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 1deaee0ed..319323b27 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -1,4 +1,4 @@ -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping/index.js'; import { replaceBy, timestamp } from '../../services/index.js'; @@ -27,14 +27,14 @@ export const createMatchboxState = ({ }; export const getMatchBoxState = - (es: AllClients) => + (es: SearchClient) => async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); return currentMetadata?.config['matchbox-state']; }; export const saveMatchBoxState = - (es: AllClients) => + (es: SearchClient) => async ({ graphqlField, projectId, diff --git a/modules/server/src/admin/schemas/ProjectSchema/utils.ts b/modules/server/src/admin/schemas/ProjectSchema/utils.ts index e17d61876..3505075d6 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/utils.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/utils.ts @@ -1,4 +1,4 @@ -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { constants } from '../../services/constants.js'; import { serializeToEsId } from '../../services/index.js'; @@ -14,7 +14,7 @@ export const newArrangerProject = (id: string): IArrangerProject => ({ timestamp: new Date().toISOString(), }); -export const getArrangerProjects = async (es: AllClients): Promise => { +export const getArrangerProjects = async (es: SearchClient): Promise => { const { body: { hits: { hits }, @@ -42,7 +42,7 @@ export const getArrangerProjects = async (es: AllClients): Promise + (es: SearchClient) => async (id: string): Promise => { //id must be lower case const _id = serializeToEsId(id); @@ -63,7 +63,7 @@ export const addArrangerProject = }; export const removeArrangerProject = - (es: AllClients) => + (es: SearchClient) => async (id: string): Promise => { const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); if (existingProject) { diff --git a/modules/server/src/admin/services/elasticsearch/index.ts b/modules/server/src/admin/services/elasticsearch/index.ts index a67bad81e..ba9720ef4 100644 --- a/modules/server/src/admin/services/elasticsearch/index.ts +++ b/modules/server/src/admin/services/elasticsearch/index.ts @@ -1,4 +1,4 @@ -import getSearchClient, { type AllClients } from '#searchClient/index.js'; +import getSearchClient, { type SearchClient } from '#searchClient/index.js'; import { type EsMapping } from './types.js'; @@ -14,7 +14,7 @@ export const createClient = async (esHost: string, esUser: string, esPass: strin }; export const getEsMapping = - (es: AllClients) => + (es: SearchClient) => async ({ esIndex, }: { diff --git a/modules/server/src/admin/types.ts b/modules/server/src/admin/types.ts index 5a04b4e23..19b6cc28b 100644 --- a/modules/server/src/admin/types.ts +++ b/modules/server/src/admin/types.ts @@ -1,7 +1,7 @@ import { type GraphQLResolveInfo } from 'graphql'; import { type MergeInfo } from 'graphql-tools'; -import { type AllClients } from '../searchClient/index.js'; +import { type SearchClient } from '../searchClient/index.js'; export type AdminApiConfig = { esHost: string; @@ -9,7 +9,7 @@ export type AdminApiConfig = { esPass: string; }; export type IQueryContext = { - es: AllClients; + es: SearchClient; }; export type ResolverOutput = T | Promise; diff --git a/modules/server/src/config/constants.ts b/modules/server/src/config/constants.ts index f0a855fa6..dcf6bd735 100644 --- a/modules/server/src/config/constants.ts +++ b/modules/server/src/config/constants.ts @@ -16,6 +16,7 @@ export const ES_INDEX = process.env.ES_INDEX || ''; export const ES_LOG = process.env.ES_LOG?.split?.(',') || 'error'; export const ES_PASS = process.env.ES_PASS || ''; export const ES_USER = process.env.ES_USER || ''; +export const SEARCH_CLIENT = process.env.SEARCH_CLIENT || ''; export const MAX_DOWNLOAD_ROWS = stringToNumber(process.env.MAX_DOWNLOAD_ROWS) || 100; export const MAX_LIVE_VERSIONS = process.env.MAX_LIVE_VERSIONS || 3; export const MAX_RESULTS_WINDOW = stringToNumber(process.env.MAX_RESULTS_WINDOW) || 10000; diff --git a/modules/server/src/config/utils/index.ts b/modules/server/src/config/utils/index.ts index 48b7c4ca0..a495d36e0 100644 --- a/modules/server/src/config/utils/index.ts +++ b/modules/server/src/config/utils/index.ts @@ -1,13 +1,13 @@ import { ENV_CONFIG } from '#config/index.js'; import { type ConfigObject, configProperties } from '#config/types.js'; import { setsMapping } from '#schema/index.js'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; export const initializeSets = async ({ esClient, setsIndex: setsIndexParam, }: { - esClient: AllClients; + esClient: SearchClient; setsIndex: string; }): Promise => { ENV_CONFIG.DEBUG_MODE && console.log(`Attempting to create Sets index "${setsIndexParam}"...`); diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index af8908d03..78ccae0b2 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -1,9 +1,9 @@ import { type GraphQLResolveInfo } from 'graphql'; -import { type AllClients } from './searchClient/index.js'; +import { type SearchClient } from './searchClient/index.js'; export type Context = { - esClient: AllClients; + esClient: SearchClient; }; export type Root = Record; diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts index 46cd11918..6a9a6d0bd 100644 --- a/modules/server/src/mapping/utils/esSearch.ts +++ b/modules/server/src/mapping/utils/esSearch.ts @@ -1,7 +1,7 @@ import { type RequestParams } from '@elastic/elasticsearch'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; -export default (esClient: AllClients) => async (params: RequestParams.Search) => { +export default (esClient: SearchClient) => async (params: RequestParams.Search) => { return esClient.search(params)?.body; }; diff --git a/modules/server/src/mapping/utils/fetchMapping.ts b/modules/server/src/mapping/utils/fetchMapping.ts index 91d837ffa..6e55cefad 100644 --- a/modules/server/src/mapping/utils/fetchMapping.ts +++ b/modules/server/src/mapping/utils/fetchMapping.ts @@ -1,8 +1,8 @@ import type { CatAliasesAliasesRecord } from '@elastic/elasticsearch/api/types'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; -export const getESAliases = async (esClient: AllClients) => { +export const getESAliases = async (esClient: SearchClient) => { const { body } = await esClient.cat.aliases({ format: 'json' }); return body; @@ -11,7 +11,7 @@ export const getESAliases = async (esClient: AllClients) => { export const checkESAlias = (aliases: CatAliasesAliasesRecord[], possibleAlias: string) => aliases?.find((foundIndex = { alias: undefined }) => foundIndex.alias === possibleAlias)?.index; -export const fetchMapping = async ({ esClient, index }: { esClient: AllClients; index: string }) => { +export const fetchMapping = async ({ esClient, index }: { esClient: SearchClient; index: string }) => { if (esClient) { console.log(`Fetching ES mapping for "${index}"...`); const aliases = await getESAliases(esClient); diff --git a/modules/server/src/schema/Root.ts b/modules/server/src/schema/Root.ts index a742581d4..db6251191 100644 --- a/modules/server/src/schema/Root.ts +++ b/modules/server/src/schema/Root.ts @@ -6,7 +6,7 @@ import Parallel from 'paralleljs'; import { ENV_CONFIG } from '#config/index.js'; import { createConnectionResolvers, saveSet, mappingToFields } from '#mapping/index.js'; import { checkESAlias, getESAliases } from '#mapping/utils/fetchMapping.js'; -import { type AllClients } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/index.js'; import { typeDefs as AggregationsTypeDefs } from './Aggregations.js'; import ConfigsTypeDefs from './configQuery.js'; @@ -82,7 +82,7 @@ export const resolvers = ({ enableAdmin, types, rootTypes, scalarTypes, getServe Date: GraphQLDate, Root: { viewer: resolveObject, - hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: AllClients }) => { + hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: SearchClient }) => { if (documentType) { if (index) { const [_, type] = types.find(([name]) => name === documentType) || []; diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts index 3980ff7b3..90d17b7e3 100644 --- a/modules/server/src/searchClient/index.ts +++ b/modules/server/src/searchClient/index.ts @@ -4,21 +4,36 @@ import { Client as OpenSearchClient, type ClientOptions as OSClientOptions } fro import { ENV_CONFIG } from '#config/index.js'; export type AllClients = ElasticClient | OpenSearchClient; -type Clients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; -type ClientTypes = keyof Clients; -type ClientOptions = { elasticsearch: ESClientOptions; opensearch: OSClientOptions }; +export type SearchClient = ReturnType; +export type SupportedClients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; +export type SupportedClientOptions = { elasticsearch: ESClientOptions; opensearch: OSClientOptions }; +export type SupportedClientTypes = keyof SupportedClients; +export type SupportedClientOptionTypes = SupportedClientOptions[keyof SupportedClientOptions]; -async function getSearchClient(options: ClientOptions[Key]) { - const { ES_HOST } = ENV_CONFIG; +const createSearchClient = ( + clientType: SupportedClientTypes, + clientOptions: SupportedClientOptionTypes, +): AllClients => { + if (clientType === 'opensearch') { + // TODO: validate + const options = clientOptions as SupportedClientOptions[typeof clientType]; + return new OpenSearchClient(options); + } else { + // TODO: validate + const options = clientOptions as SupportedClientOptions[typeof clientType]; + return new ElasticClient(options); + } +}; + +async function getSearchClient(options: SupportedClientOptionTypes) { + const { ES_HOST, SEARCH_CLIENT } = ENV_CONFIG; const searchConfig = await (await fetch(ES_HOST)).json(); const { distribution } = searchConfig.version; try { - if (distribution === 'opensearch') { - const clientOptions = options as OSClientOptions; - return new OpenSearchClient(clientOptions); + if (distribution === 'opensearch' || SEARCH_CLIENT === 'opensearch') { + return createSearchClient('opensearch', options); } else { - const clientOptions = options as ESClientOptions; - return new ElasticClient(clientOptions); + return createSearchClient('elasticsearch', options); } } catch (error) { console.error(error); @@ -26,6 +41,4 @@ async function getSearchClient(options: ClientOptions[K } } -export type SearchClientType = ReturnType; - export default getSearchClient; From 92b3f74ceed83ef4da2df3334cc7cd6862a00506 Mon Sep 17 00:00:00 2001 From: Dan DeMaria Date: Wed, 11 Feb 2026 15:08:49 -0500 Subject: [PATCH 7/7] Separate Types file & update imports --- modules/server/src/admin/index.ts | 2 +- .../src/admin/schemas/AggsState/utils.ts | 3 ++- .../src/admin/schemas/ColumnsState/utils.ts | 2 +- .../admin/schemas/ExtendedMapping/utils.ts | 2 +- .../src/admin/schemas/IndexSchema/utils.ts | 2 +- .../src/admin/schemas/MatchboxState/utils.ts | 2 +- .../src/admin/schemas/ProjectSchema/utils.ts | 2 +- .../src/admin/services/elasticsearch/index.ts | 3 ++- modules/server/src/admin/types.ts | 2 +- modules/server/src/config/utils/index.ts | 2 +- modules/server/src/gqlServer.ts | 2 +- modules/server/src/mapping/utils/esSearch.ts | 2 +- .../server/src/mapping/utils/fetchMapping.ts | 2 +- modules/server/src/schema/Root.ts | 2 +- modules/server/src/searchClient/index.ts | 21 +++++++------------ modules/server/src/searchClient/types.ts | 11 ++++++++++ 16 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 modules/server/src/searchClient/types.ts diff --git a/modules/server/src/admin/index.ts b/modules/server/src/admin/index.ts index be50a5d76..e57de4d12 100644 --- a/modules/server/src/admin/index.ts +++ b/modules/server/src/admin/index.ts @@ -5,7 +5,7 @@ import { ApolloServer } from 'apollo-server-express'; import { type GraphQLSchema } from 'graphql'; import { print } from 'graphql/language/printer'; -import { type SearchClient } from '../searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { createAggsStateByIndexResolver, diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index e034cae4c..f49a78aa4 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -1,7 +1,8 @@ import { sortBy } from 'ramda'; +import { type SearchClient } from '#searchClient/types.js'; + import { mappingToAggsState } from '../../../mapping/index.js'; -import { type SearchClient } from '../../../searchClient/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; import { timestamp } from '../../services/index.js'; import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index e644cd181..9d07c33f7 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -1,6 +1,6 @@ import { sortBy } from 'ramda'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { mappingToColumnsState } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index 15a463a31..d64ba530b 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -1,6 +1,6 @@ import { UserInputError } from 'apollo-server'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { extendMapping } from '../../../mapping/index.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index df061120d..29af66d3c 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -1,7 +1,7 @@ import { UserInputError } from 'apollo-server'; import Qew from 'qew'; // TODO: using 0.9.13 because later versions break the async -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { constants } from '../../services/constants.js'; import { getEsMapping } from '../../services/elasticsearch/index.js'; diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 319323b27..71b179eb4 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -1,4 +1,4 @@ -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping/index.js'; import { replaceBy, timestamp } from '../../services/index.js'; diff --git a/modules/server/src/admin/schemas/ProjectSchema/utils.ts b/modules/server/src/admin/schemas/ProjectSchema/utils.ts index 3505075d6..f51a88b9e 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/utils.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/utils.ts @@ -1,4 +1,4 @@ -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { constants } from '../../services/constants.js'; import { serializeToEsId } from '../../services/index.js'; diff --git a/modules/server/src/admin/services/elasticsearch/index.ts b/modules/server/src/admin/services/elasticsearch/index.ts index ba9720ef4..1f19239ad 100644 --- a/modules/server/src/admin/services/elasticsearch/index.ts +++ b/modules/server/src/admin/services/elasticsearch/index.ts @@ -1,4 +1,5 @@ -import getSearchClient, { type SearchClient } from '#searchClient/index.js'; +import getSearchClient from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { type EsMapping } from './types.js'; diff --git a/modules/server/src/admin/types.ts b/modules/server/src/admin/types.ts index 19b6cc28b..1ce511792 100644 --- a/modules/server/src/admin/types.ts +++ b/modules/server/src/admin/types.ts @@ -1,7 +1,7 @@ import { type GraphQLResolveInfo } from 'graphql'; import { type MergeInfo } from 'graphql-tools'; -import { type SearchClient } from '../searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; export type AdminApiConfig = { esHost: string; diff --git a/modules/server/src/config/utils/index.ts b/modules/server/src/config/utils/index.ts index a495d36e0..5ccdaabee 100644 --- a/modules/server/src/config/utils/index.ts +++ b/modules/server/src/config/utils/index.ts @@ -1,7 +1,7 @@ import { ENV_CONFIG } from '#config/index.js'; import { type ConfigObject, configProperties } from '#config/types.js'; import { setsMapping } from '#schema/index.js'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; export const initializeSets = async ({ esClient, diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index 78ccae0b2..62d44d985 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -1,6 +1,6 @@ import { type GraphQLResolveInfo } from 'graphql'; -import { type SearchClient } from './searchClient/index.js'; +import { type SearchClient } from './searchClient/types.js'; export type Context = { esClient: SearchClient; diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts index 6a9a6d0bd..4677f423b 100644 --- a/modules/server/src/mapping/utils/esSearch.ts +++ b/modules/server/src/mapping/utils/esSearch.ts @@ -1,6 +1,6 @@ import { type RequestParams } from '@elastic/elasticsearch'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; export default (esClient: SearchClient) => async (params: RequestParams.Search) => { return esClient.search(params)?.body; diff --git a/modules/server/src/mapping/utils/fetchMapping.ts b/modules/server/src/mapping/utils/fetchMapping.ts index 6e55cefad..21b7bd2b7 100644 --- a/modules/server/src/mapping/utils/fetchMapping.ts +++ b/modules/server/src/mapping/utils/fetchMapping.ts @@ -1,6 +1,6 @@ import type { CatAliasesAliasesRecord } from '@elastic/elasticsearch/api/types'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; export const getESAliases = async (esClient: SearchClient) => { const { body } = await esClient.cat.aliases({ format: 'json' }); diff --git a/modules/server/src/schema/Root.ts b/modules/server/src/schema/Root.ts index db6251191..2c52080b6 100644 --- a/modules/server/src/schema/Root.ts +++ b/modules/server/src/schema/Root.ts @@ -6,7 +6,7 @@ import Parallel from 'paralleljs'; import { ENV_CONFIG } from '#config/index.js'; import { createConnectionResolvers, saveSet, mappingToFields } from '#mapping/index.js'; import { checkESAlias, getESAliases } from '#mapping/utils/fetchMapping.js'; -import { type SearchClient } from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; import { typeDefs as AggregationsTypeDefs } from './Aggregations.js'; import ConfigsTypeDefs from './configQuery.js'; diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts index 90d17b7e3..4e6c1ee23 100644 --- a/modules/server/src/searchClient/index.ts +++ b/modules/server/src/searchClient/index.ts @@ -1,31 +1,26 @@ -import { Client as ElasticClient, type ClientOptions as ESClientOptions } from '@elastic/elasticsearch'; -import { Client as OpenSearchClient, type ClientOptions as OSClientOptions } from '@opensearch-project/opensearch'; +import { Client as ElasticClient } from '@elastic/elasticsearch'; +import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; import { ENV_CONFIG } from '#config/index.js'; -export type AllClients = ElasticClient | OpenSearchClient; -export type SearchClient = ReturnType; -export type SupportedClients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; -export type SupportedClientOptions = { elasticsearch: ESClientOptions; opensearch: OSClientOptions }; -export type SupportedClientTypes = keyof SupportedClients; -export type SupportedClientOptionTypes = SupportedClientOptions[keyof SupportedClientOptions]; +import type { AllClients, SupportedClientTypes, SupportedClientOptionTypes, SupportedClientOptions } from './types.js'; -const createSearchClient = ( +export const createSearchClient = ( clientType: SupportedClientTypes, clientOptions: SupportedClientOptionTypes, ): AllClients => { if (clientType === 'opensearch') { - // TODO: validate + // TODO: validate / no use of 'as' const options = clientOptions as SupportedClientOptions[typeof clientType]; return new OpenSearchClient(options); } else { - // TODO: validate + // TODO: validate / no use of 'as' const options = clientOptions as SupportedClientOptions[typeof clientType]; return new ElasticClient(options); } }; -async function getSearchClient(options: SupportedClientOptionTypes) { +export default async function getSearchClient(options: SupportedClientOptionTypes) { const { ES_HOST, SEARCH_CLIENT } = ENV_CONFIG; const searchConfig = await (await fetch(ES_HOST)).json(); const { distribution } = searchConfig.version; @@ -40,5 +35,3 @@ async function getSearchClient(options: SupportedClientOptionTypes) { throw new Error(`Error configuring Search Client`); } } - -export default getSearchClient; diff --git a/modules/server/src/searchClient/types.ts b/modules/server/src/searchClient/types.ts new file mode 100644 index 000000000..c4409cbca --- /dev/null +++ b/modules/server/src/searchClient/types.ts @@ -0,0 +1,11 @@ +import type { Client as ElasticClient, ClientOptions as ESClientOptions } from '@elastic/elasticsearch'; +import type { Client as OpenSearchClient, ClientOptions as OSClientOptions } from '@opensearch-project/opensearch'; + +import { type createSearchClient } from './index.js'; + +export type AllClients = ElasticClient | OpenSearchClient; +export type SearchClient = ReturnType; +export type SupportedClients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; +export type SupportedClientOptions = { elasticsearch: ESClientOptions; opensearch: OSClientOptions }; +export type SupportedClientTypes = keyof SupportedClients; +export type SupportedClientOptionTypes = SupportedClientOptions[keyof SupportedClientOptions];