diff --git a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts index be3f3cbfa..d3803eee1 100644 --- a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts +++ b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts @@ -36,6 +36,7 @@ import Sentry from '@sentry/minimal'; import { processRowsUtil } from '../utils/process-found-rows-util.js'; import JSON5 from 'json5'; import { buildActionEventDto } from '../../table-actions/table-action-rules-module/utils/build-found-action-event-dto.util.js'; +import { TableSettingsDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table-settings.ds.js'; @Injectable() export class GetTableRowsUseCase extends AbstractUseCase implements IGetTableRows { @@ -231,34 +232,38 @@ export class GetTableRowsUseCase extends AbstractUseCase 0) { - identities = await Promise.all( - tableForeignKeys.map(async (foreignKey) => { - const foreignKeysValuesCollection = rowsRO.rows - .filter((row) => row[foreignKey.column_name]) - .map((row) => row[foreignKey.column_name]) as (string | number)[]; - - const foreignTableSettings = await this._dbContext.tableSettingsRepository.findTableSettings( - connectionId, - foreignKey.referenced_table_name, - ); - - const identityColumns = await dao.getIdentityColumns( - foreignKey.referenced_table_name, - foreignKey.referenced_column_name, - foreignTableSettings?.identity_column, - foreignKeysValuesCollection, - userEmail, - ); - - return { + for (const foreignKey of tableForeignKeys) { + const foreignKeysValuesCollection = rowsRO.rows + .filter((row) => row[foreignKey.column_name]) + .map((row) => row[foreignKey.column_name]) as (string | number)[]; + + const foreignTableSettings = await this._dbContext.tableSettingsRepository.findTableSettings( + connectionId, + foreignKey.referenced_table_name, + ); + + const identityColumns = await this.getBatchedIdentityColumns( + foreignKeysValuesCollection, + foreignKey, + dao, + foreignTableSettings, + userEmail, + ); + + if (identities.findIndex((el) => el.referenced_table_name === foreignKey.referenced_table_name) > -1) { + identities + .find((el) => el.referenced_table_name === foreignKey.referenced_table_name) + .identity_columns.push(...identityColumns); + } else { + identities.push({ referenced_table_name: foreignKey.referenced_table_name, identity_columns: identityColumns, - }; - }), - ); + }); + } + } } const foreignKeysConformity = tableForeignKeys.map((key) => ({ @@ -276,15 +281,11 @@ export class GetTableRowsUseCase extends AbstractUseCase el[element.realFKeyName] === row[element.currentFKeyName], ); - row[element.currentFKeyName] = foundIdentityForCurrentValue ? { ...foundIdentityForCurrentValue } : {}; }); }); operationResult = OperationResultStatusEnum.successfully; - if (connectionId === 'JYCEZYqk' && tableName === 'followers') { - console.log('PROCESS ROWS ON LAST STAGE BEFORE RETURNING TO USER ', JSON.stringify(rowsRO.rows)); - } return rowsRO; } catch (e) { Sentry.captureException(e); @@ -344,4 +345,38 @@ export class GetTableRowsUseCase extends AbstractUseCase(array: T[], chunkSize: number): T[][] { + const results = []; + for (let i = 0; i < array.length; i += chunkSize) { + results.push(array.slice(i, i + chunkSize)); + } + return results; + } + + private async getBatchedIdentityColumns( + foreignKeysValuesCollection: Array, + foreignKey: ForeignKeyDS, + dao: IDataAccessObject | IDataAccessObjectAgent, + foreignTableSettings: TableSettingsDS, + userEmail: string, + ): Promise>> { + foreignKeysValuesCollection = [...new Set(foreignKeysValuesCollection)]; + const batchSize = 50; + const chunkedValues = this.chunkArray(foreignKeysValuesCollection, batchSize); + let identityColumns = []; + + for (const chunk of chunkedValues) { + const result = await dao.getIdentityColumns( + foreignKey.referenced_table_name, + foreignKey.referenced_column_name, + foreignTableSettings?.identity_column, + chunk, + userEmail, + ); + identityColumns = identityColumns.concat(result); + } + + return identityColumns; + } } diff --git a/backend/test/ava-tests/saas-tests/different-tables-structures-e2e-test.ts b/backend/test/ava-tests/saas-tests/different-tables-structures-e2e-test.ts new file mode 100644 index 000000000..c01001592 --- /dev/null +++ b/backend/test/ava-tests/saas-tests/different-tables-structures-e2e-test.ts @@ -0,0 +1,246 @@ +/* eslint-disable security/detect-non-literal-fs-filename */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable security/detect-object-injection */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { LogOperationTypeEnum, QueryOrderingEnum } from '../../../src/enums/index.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { Messages } from '../../../src/exceptions/text/messages.js'; +import { Constants } from '../../../src/helpers/constants/constants.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { createTestTable } from '../../utils/create-test-table.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { registerUserAndReturnUserInfo } from '../../utils/register-user-and-return-user-info.js'; +import { TestUtils } from '../../utils/test.utils.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { ValidationError } from 'class-validator'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { join } from 'path'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { clearAllTestKnex, getTestKnex } from '../../utils/get-test-knex.js'; +import { Knex } from 'knex'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const mockFactory = new MockFactory(); +let app: INestApplication; +let testUtils: TestUtils; +const testSearchedUserName = 'Vasia'; +const testTables: Array = []; +let currentTest; + +test.before(async () => { + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }).compile(); + app = moduleFixture.createNestApplication() as any; + testUtils = moduleFixture.get(TestUtils); + + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter()); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + app.getHttpServer().listen(0); + await loadTestData(); +}); + +test.after(async () => { + try { + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +const testUsersTableName = 'users'; +const testTransactionsTableName = 'transactions'; + +const createUsersTableRawQuery = ` +CREATE TABLE users ( + userId SERIAL PRIMARY KEY, + username VARCHAR(50) NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() +);`; + +const createTransactionsTableRawQuery = `CREATE TABLE transactions ( + buyerId INT, + reviewerId INT, + transaction_date DATE NOT NULL, + description TEXT, + amount NUMERIC(10, 2), + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (buyerId, reviewerId), + FOREIGN KEY (buyerId) REFERENCES users(userId), + FOREIGN KEY (reviewerId) REFERENCES users(userId) +);`; + +const testEntitiesCount = 500; + +async function fillUsersTable(knex: Knex): Promise { + for (let i = 0; i < testEntitiesCount; i++) { + await knex('users').insert({ username: faker.person.firstName() }); + } +} + +const insertedIdCombos: Array<{ buyerid: number; reviewerid: number }> = []; + +async function fillTransactionsTable(knex: Knex): Promise { + for (let i = 0; i < testEntitiesCount; i++) { + const { buyerid, reviewerid } = getUniqueIdsComboForInsert(); + await knex('transactions').insert({ + buyerid: buyerid, + reviewerid: reviewerid, + transaction_date: faker.date.past(), + description: faker.lorem.sentence(), + amount: faker.number.int({ min: 100, max: 10000 }), + }); + } +} + +function getUniqueIdsCombo(): { buyerid: number; reviewerid: number } { + return { + buyerid: faker.number.int({ min: 1, max: testEntitiesCount }), + reviewerid: faker.number.int({ min: 1, max: testEntitiesCount }), + }; +} + +function getUniqueIdsComboForInsert(): { buyerid: number; reviewerid: number } { + if (insertedIdCombos.length >= testEntitiesCount) { + return insertedIdCombos.shift(); + } + while (true) { + const { buyerid, reviewerid } = getUniqueIdsCombo(); + if (!insertedIdCombos.find((idCombo) => idCombo.buyerid === buyerid && idCombo.reviewerid === reviewerid)) { + insertedIdCombos.push({ buyerid, reviewerid }); + return { buyerid, reviewerid }; + } + } +} + +async function loadTestData(): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgres; + const testKnex = getTestKnex(connectionToTestDB); + await testKnex.raw(`DROP TABLE IF EXISTS transactions;`); + await testKnex.raw(`DROP TABLE IF EXISTS users;`); + await testKnex.raw(createUsersTableRawQuery); + await testKnex.raw(createTransactionsTableRawQuery); + await fillUsersTable(testKnex); + await fillTransactionsTable(testKnex); + await clearAllTestKnex(); +} + +currentTest = 'GET /connection/tables/:slug'; +test.serial(`${currentTest} should return list of tables in connection`, async (t) => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgres; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const getTablesResponse = await request(app.getHttpServer()) + .get(`/connection/tables/${createConnectionRO.id}`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const getTablesRO = JSON.parse(getTablesResponse.text); + t.is(typeof getTablesRO, 'object'); + t.is(getTablesRO.length > 0, true); + } catch (error) { + console.error(error); + throw error; + } +}); + +currentTest = 'GET /table/rows/:slug'; + +test.serial(`${currentTest} should return list of rows of the tables`, async (t) => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgres; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const foundRowsFromUsersTableResponse = await request(app.getHttpServer()) + .get(`/table/rows/${createConnectionRO.id}?tableName=${testUsersTableName}&page=1&perPage=500`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const foundRowsFromUsersTableRO = JSON.parse(foundRowsFromUsersTableResponse.text); + t.is(typeof foundRowsFromUsersTableRO, 'object'); + t.is(foundRowsFromUsersTableRO.hasOwnProperty('rows'), true); + t.is(foundRowsFromUsersTableRO.hasOwnProperty('primaryColumns'), true); + t.is(foundRowsFromUsersTableRO.hasOwnProperty('pagination'), true); + t.is(foundRowsFromUsersTableRO.rows.length, 500); + t.is(Object.keys(foundRowsFromUsersTableRO.rows[1]).length, 4); + t.is(foundRowsFromUsersTableRO.rows[0].hasOwnProperty('userid'), true); + t.is(foundRowsFromUsersTableRO.rows[1].hasOwnProperty('username'), true); + t.is(foundRowsFromUsersTableRO.rows[2].hasOwnProperty('created_at'), true); + + const foundRowsFromTransactionsTableResponse = await request(app.getHttpServer()) + .get(`/table/rows/${createConnectionRO.id}?tableName=${testTransactionsTableName}&page=1&perPage=500`) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const foundRowsFromTransactionsTableRO = JSON.parse(foundRowsFromTransactionsTableResponse.text); + t.is(typeof foundRowsFromTransactionsTableRO, 'object'); + t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('rows'), true); + t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('primaryColumns'), true); + t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('pagination'), true); + t.is(foundRowsFromTransactionsTableRO.rows.length, 500); + t.is(Object.keys(foundRowsFromTransactionsTableRO.rows[1]).length, 7); + t.is(foundRowsFromTransactionsTableRO.rows[0].hasOwnProperty('buyerid'), true); + t.is(foundRowsFromTransactionsTableRO.rows[1].hasOwnProperty('reviewerid'), true); + t.is(foundRowsFromTransactionsTableRO.rows[2].hasOwnProperty('transaction_date'), true); + + for (const row of foundRowsFromTransactionsTableRO.rows) { + t.is(row.hasOwnProperty('buyerid'), true); + const buyerId = row.buyerid; + t.is(typeof buyerId, 'object'); + t.is(buyerId.hasOwnProperty('userid'), true); + t.is(typeof buyerId.userid, 'number'); + const reviewerId = row.reviewerid; + t.is(typeof reviewerId, 'object'); + t.is(reviewerId.hasOwnProperty('userid'), true); + t.is(typeof reviewerId.userid, 'number'); + } + } catch (error) { + console.error(error); + throw error; + } +});