From 4db3c75df41c14c9c73aa4096265626ca8c5ccd0 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Wed, 26 Feb 2025 11:11:31 +0000 Subject: [PATCH 1/2] Enhance OracleDB data access with date/time type detection and proper formatting in queries for better filtering capabilities --- .../saas-tests/table-oracledb-e2e.test.ts | 93 +++++++++++++++++-- .../basic-data-access-object.ts | 38 ++++++++ .../data-access-object-oracle.ts | 74 ++++++++++++--- shared-code/src/helpers/is-database-date.ts | 23 ++++- 4 files changed, 206 insertions(+), 22 deletions(-) diff --git a/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts index 212cf3799..afe8a54f6 100644 --- a/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts @@ -4,11 +4,16 @@ import { faker } from '@faker-js/faker'; import { INestApplication, ValidationPipe } from '@nestjs/common'; import { Test } from '@nestjs/testing'; import test from 'ava'; +import { ValidationError } from 'class-validator'; import cookieParser from 'cookie-parser'; +import fs from 'fs'; +import path, { join } from 'path'; import request from 'supertest'; +import { fileURLToPath } from 'url'; import { ApplicationModule } from '../../../src/app.module.js'; import { LogOperationTypeEnum, QueryOrderingEnum } from '../../../src/enums/index.js'; import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; import { Messages } from '../../../src/exceptions/text/messages.js'; import { Cacher } from '../../../src/helpers/cache/cacher.js'; import { Constants } from '../../../src/helpers/constants/constants.js'; @@ -20,14 +25,6 @@ import { dropTestTables } from '../../utils/drop-test-tables.js'; import { getTestData } from '../../utils/get-test-data.js'; import { registerUserAndReturnUserInfo } from '../../utils/register-user-and-return-user-info.js'; import { TestUtils } from '../../utils/test.utils.js'; -import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; -import { ValidationError } from 'class-validator'; -import knex, { Knex } from 'knex'; -import fs from 'fs'; -import path from 'path'; -import { fileURLToPath } from 'url'; -import { join } from 'path'; -import oracledb from 'oracledb'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -1349,6 +1346,86 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC }, ); +test.only(`${currentTest} with pagination, with sorting and with filtering by date fields +should return all found rows with search, pagination: page=1, perPage=2 and DESC sorting and filtering`, async (t) => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToOracleDB; + const firstUserToken = (await registerUserAndReturnUserInfo(app)).token; + const { testTableName, testTableColumnName } = await createTestOracleTable(connectionToTestDB); + + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + const createConnectionRO = JSON.parse(createConnectionResponse.text); + t.is(createConnectionResponse.status, 201); + + const createTableSettingsDTO = mockFactory.generateTableSettings( + createConnectionRO.id, + testTableName, + [testTableColumnName], + undefined, + undefined, + 3, + QueryOrderingEnum.DESC, + 'id', + undefined, + undefined, + undefined, + undefined, + undefined, + ); + + const firstFieldName = 'created_at'; + const secondFieldName = 'updated_at'; + const firstFieldValue = "2011-11-03"; + + const filters = { + [firstFieldName]: { lt: firstFieldValue }, + }; + + // const filters = { + // id: { lt: 30 }, + // }; + + const getTableRowsResponse = await request(app.getHttpServer()) + .post(`/table/rows/find/${createConnectionRO.id}?tableName=${testTableName}&page=1&perPage=2`) + .send({ filters }) + .set('Cookie', firstUserToken) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const getTableRowsRO = JSON.parse(getTableRowsResponse.text); + console.log('🚀 ~ getTableRowsRO:', getTableRowsRO); + t.is(getTableRowsResponse.status, 201); + t.is(typeof getTableRowsRO, 'object'); + t.is(getTableRowsRO.hasOwnProperty('rows'), true); + t.is(getTableRowsRO.hasOwnProperty('primaryColumns'), true); + t.is(getTableRowsRO.hasOwnProperty('pagination'), true); + t.is(getTableRowsRO.rows.length, 2); + t.is(Object.keys(getTableRowsRO.rows[1]).length, 5); + + t.is(getTableRowsRO.rows[0][testTableColumnName], testSearchedUserName); + t.is(getTableRowsRO.rows[0].id, 38); + t.is(getTableRowsRO.rows[1][testTableColumnName], testSearchedUserName); + t.is(getTableRowsRO.rows[1].id, 22); + + t.is(getTableRowsRO.pagination.currentPage, 1); + t.is(getTableRowsRO.pagination.perPage, 2); + t.is(typeof getTableRowsRO.primaryColumns, 'object'); + t.is(getTableRowsRO.primaryColumns[0].hasOwnProperty('column_name'), true); + + // t.is(getTableRowsRO.primaryColumns[0].hasOwnProperty('data_type'), true); + } catch (e) { + console.error(e); + throw e; + } +}); + test.serial( `${currentTest} with search, with pagination, with sorting and with filtering should return all found rows with search, pagination: page=1, perPage=10 and DESC sorting and filtering'`, diff --git a/shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts b/shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts index 306f8bf7c..389fdf699 100644 --- a/shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts +++ b/shared-code/src/data-access-layer/data-access-objects/basic-data-access-object.ts @@ -72,6 +72,44 @@ export class BasicDataAccessObject { } } + protected isDateTimeType(columnTypeName: string): boolean { + const dateTimeDataTypes = [ + // PostgreSQL + 'DATE', + 'TIME', + 'TIMETZ', + 'TIMESTAMP', + 'TIMESTAMPTZ', + + // MySQL + 'DATE', + 'DATETIME', + 'TIMESTAMP', + 'TIME', + 'YEAR', + + // MS SQL Server + 'DATE', + 'DATETIME', + 'DATETIME2', + 'DATETIMEOFFSET', + 'SMALLDATETIME', + 'TIME', + + // OracleDB + 'DATE', + 'TIMESTAMP', + 'TIMESTAMP WITH TIME ZONE', + 'TIMESTAMP WITH LOCAL TIME ZONE', + + // IBM Db2 + 'DATE', + 'TIME', + 'TIMESTAMP', + ]; + return dateTimeDataTypes.includes(columnTypeName.toUpperCase()); + } + private isValidName(name: string): boolean { return typeof name === 'string' && name.length > 0 && /^[a-zA-Z0-9_]+$/.test(name); } diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts index fff3e37c8..60c0c976f 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts @@ -1,7 +1,20 @@ /* eslint-disable security/detect-object-injection */ +import * as csv from 'csv'; import { Knex } from 'knex'; +import { Readable, Stream } from 'node:stream'; +import { LRUStorage } from '../../caching/lru-storage.js'; import { checkFieldAutoincrement } from '../../helpers/check-field-autoincrement.js'; import { DAO_CONSTANTS } from '../../helpers/data-access-objects-constants.js'; +import { ERROR_MESSAGES } from '../../helpers/errors/error-messages.js'; +import { + isOracleDateOrTimeType, + isOracleDateStringByRegexp, + isOracleDateType, + isOracleTimeType, +} from '../../helpers/is-database-date.js'; +import { objectKeysToLowercase } from '../../helpers/object-kyes-to-lowercase.js'; +import { renameObjectKeyName } from '../../helpers/rename-object-keyname.js'; +import { tableSettingsFieldValidator } from '../../helpers/validation/table-settings-validator.js'; import { AutocompleteFieldsDS } from '../shared/data-structures/autocomplete-fields.ds.js'; import { ConnectionParams } from '../shared/data-structures/connections-params.ds.js'; import { FilteringFieldsDS } from '../shared/data-structures/filtering-fields.ds.js'; @@ -11,20 +24,12 @@ import { PrimaryKeyDS } from '../shared/data-structures/primary-key.ds.js'; import { ReferencedTableNamesAndColumnsDS } from '../shared/data-structures/referenced-table-names-columns.ds.js'; import { TableSettingsDS } from '../shared/data-structures/table-settings.ds.js'; import { TableStructureDS } from '../shared/data-structures/table-structure.ds.js'; +import { TableDS } from '../shared/data-structures/table.ds.js'; import { TestConnectionResultDS } from '../shared/data-structures/test-result-connection.ds.js'; import { ValidateTableSettingsDS } from '../shared/data-structures/validate-table-settings.ds.js'; import { FilterCriteriaEnum } from '../shared/enums/filter-criteria.enum.js'; import { IDataAccessObject } from '../shared/interfaces/data-access-object.interface.js'; import { BasicDataAccessObject } from './basic-data-access-object.js'; -import { LRUStorage } from '../../caching/lru-storage.js'; -import { objectKeysToLowercase } from '../../helpers/object-kyes-to-lowercase.js'; -import { renameObjectKeyName } from '../../helpers/rename-object-keyname.js'; -import { tableSettingsFieldValidator } from '../../helpers/validation/table-settings-validator.js'; -import { TableDS } from '../shared/data-structures/table.ds.js'; -import { ERROR_MESSAGES } from '../../helpers/errors/error-messages.js'; -import { Stream, Readable } from 'node:stream'; -import * as csv from 'csv'; -import { isOracleDateOrTimeType, isOracleDateStringByRegexp } from '../../helpers/is-database-date.js'; type RefererencedConstraint = { TABLE_NAME: string; @@ -232,6 +237,13 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa const tableStructure = await this.getTableStructure(tableName); const availableFields = this.findAvailableFields(settings, tableStructure); + const timestampColumnNames = tableStructure + .filter(({ data_type }) => isOracleTimeType(data_type)) + .map(({ column_name }) => column_name); + + const datesColumnsNames = tableStructure + .filter(({ data_type }) => isOracleDateType(data_type)) + .map(({ column_name }) => column_name); const searchedFields = settings?.search_fields?.length > 0 ? settings.search_fields : searchedFieldValue ? availableFields : []; @@ -248,6 +260,8 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa searchedFieldValue, filteringFields, settings, + timestampColumnNames, + datesColumnsNames, ); } @@ -260,8 +274,10 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa availableFields: Array, searchedFields: Array, searchedFieldValue: any, - filteringFields: any, + filteringFields: FilteringFieldsDS[], settings: TableSettingsDS, + timestampColumnNames: Array, + datesColumnsNames: Array, ) { const offset = (page - 1) * perPage; @@ -277,9 +293,21 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa } }; - const applyFilteringFields = (builder: Knex.QueryBuilder) => { + const applyFilteringFields = ( + builder: Knex.QueryBuilder, + timestampColumnNames: Array, + datesColumnsNames: Array, + ) => { if (filteringFields && filteringFields.length > 0) { - for (const { field, criteria, value } of filteringFields) { + // eslint-disable-next-line prefer-const + for (let { field, criteria, value } of filteringFields) { + if (datesColumnsNames.includes(field)) { + const valueToDate = new Date(String(value)); + value = this.formatDate(valueToDate); + } + if (timestampColumnNames.includes(field)) { + value = this.formatTimestamp(String(value)); + } const operators = { [FilterCriteriaEnum.eq]: '=', [FilterCriteriaEnum.startswith]: 'like', @@ -314,7 +342,7 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa .withSchema(tableSchema) .select(availableFields) .modify(applySearchFields) - .modify(applyFilteringFields) + .modify((builder) => applyFilteringFields(builder, timestampColumnNames, datesColumnsNames)) .modify(applyOrdering) .limit(perPage) .offset(offset); @@ -770,4 +798,24 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa const resultString = `${day}-${monthNames[monthIndex]}-${year}`; return resultString; } + + private formatTimestamp(timestamp: string | number | Date): string { + const date = new Date(timestamp); + + const day = `0${date.getDate()}`.slice(-2); + const monthNames = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']; + const month = monthNames[date.getMonth()]; + const year = date.getFullYear().toString().slice(-2); + + let hours = date.getHours(); + const period = hours >= 12 ? 'PM' : 'AM'; + hours = hours % 12; + hours = hours ? hours : 12; + const hoursStr = `0${hours}`.slice(-2); + + const minutes = `0${date.getMinutes()}`.slice(-2); + const seconds = `0${date.getSeconds()}`.slice(-2); + + return `${day}-${month}-${year} ${hoursStr}:${minutes}:${seconds} ${period}`; + } } diff --git a/shared-code/src/helpers/is-database-date.ts b/shared-code/src/helpers/is-database-date.ts index 862e79f8c..c95714031 100644 --- a/shared-code/src/helpers/is-database-date.ts +++ b/shared-code/src/helpers/is-database-date.ts @@ -18,10 +18,31 @@ export function isOracleDateOrTimeType(type: string): boolean { if (type.toLowerCase().includes('timestamp')) { return true; } - const dateTypes = ['date', 'timestamp', 'timestamp with time zone', 'timestamp with local time zone']; + const dateTypes = [ + 'date', + 'timestamp', + 'timestamp with time zone', + 'timestamp with local time zone', + 'timestamp(6) with local time zone', + 'timestamp(0) with local time zone', + ]; return dateTypes.includes(type.toLowerCase()); } +export function isOracleTimeType(type: string): boolean { + return [ + 'timestamp', + 'timestamp with time zone', + 'timestamp with local time zone', + 'timestamp(6) with local time zone', + 'timestamp(0) with local time zone', + ].includes(type.toLowerCase()); +} + +export function isOracleDateType(type: string): boolean { + return ['date'].includes(type.toLowerCase()); +} + export function isOracleDateStringByRegexp(value: string): boolean { const dateRegexp = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/; return dateRegexp.test(value); From 53b7f849c83c601d0559a99e9eb8ca872e1b304f Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Wed, 26 Feb 2025 12:12:01 +0000 Subject: [PATCH 2/2] Fix test case for pagination and filtering by updating expected ID and adjusting created_at date in test setup --- .../test/ava-tests/saas-tests/table-oracledb-e2e.test.ts | 9 ++------- backend/test/utils/create-test-table.ts | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts index afe8a54f6..0e5960045 100644 --- a/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-oracledb-e2e.test.ts @@ -1346,7 +1346,7 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC }, ); -test.only(`${currentTest} with pagination, with sorting and with filtering by date fields +test.serial(`${currentTest} with pagination, with sorting and with filtering by date fields should return all found rows with search, pagination: page=1, perPage=2 and DESC sorting and filtering`, async (t) => { try { const connectionToTestDB = getTestData(mockFactory).connectionToOracleDB; @@ -1388,10 +1388,6 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC [firstFieldName]: { lt: firstFieldValue }, }; - // const filters = { - // id: { lt: 30 }, - // }; - const getTableRowsResponse = await request(app.getHttpServer()) .post(`/table/rows/find/${createConnectionRO.id}?tableName=${testTableName}&page=1&perPage=2`) .send({ filters }) @@ -1400,7 +1396,6 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC .set('Accept', 'application/json'); const getTableRowsRO = JSON.parse(getTableRowsResponse.text); - console.log('🚀 ~ getTableRowsRO:', getTableRowsRO); t.is(getTableRowsResponse.status, 201); t.is(typeof getTableRowsRO, 'object'); t.is(getTableRowsRO.hasOwnProperty('rows'), true); @@ -1410,7 +1405,7 @@ should return all found rows with search, pagination: page=1, perPage=2 and DESC t.is(Object.keys(getTableRowsRO.rows[1]).length, 5); t.is(getTableRowsRO.rows[0][testTableColumnName], testSearchedUserName); - t.is(getTableRowsRO.rows[0].id, 38); + t.is(getTableRowsRO.rows[0].id, 1); t.is(getTableRowsRO.rows[1][testTableColumnName], testSearchedUserName); t.is(getTableRowsRO.rows[1].id, 22); diff --git a/backend/test/utils/create-test-table.ts b/backend/test/utils/create-test-table.ts index 09d013f00..107907ee9 100644 --- a/backend/test/utils/create-test-table.ts +++ b/backend/test/utils/create-test-table.ts @@ -325,7 +325,7 @@ export async function createTestOracleTable( [pColumnName]: ++counter, [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), + created_at: new Date("2010-11-03"), updated_at: new Date(), }); } else {