From 06860a0c3d4a46f02741af5ad3a91447a7acc59a Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Tue, 16 Jan 2024 10:53:37 +0100 Subject: [PATCH 1/7] ChangeSet --- .changeset/spotty-queens-greet.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/spotty-queens-greet.md diff --git a/.changeset/spotty-queens-greet.md b/.changeset/spotty-queens-greet.md new file mode 100644 index 000000000..63c7031c2 --- /dev/null +++ b/.changeset/spotty-queens-greet.md @@ -0,0 +1,5 @@ +--- +'@wbce-d9/api': minor +--- + +fix cache compression From 7f0e16f2c302252c47dbfbf27c874f29772ffa9b Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Mon, 5 Feb 2024 16:24:41 +0100 Subject: [PATCH 2/7] fix: parse cache if necessary & send cache as buffer --- api/src/cache.ts | 6 +++--- api/src/middleware/cache.ts | 6 ++++-- api/src/utils/get-permissions.ts | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/api/src/cache.ts b/api/src/cache.ts index 9eead92c4..6ec7f26fa 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -110,7 +110,7 @@ export async function setSystemCache(key: string, value: any, ttl?: number): Pro export async function getSystemCache(key: string): Promise> { const { systemCache } = getCache(); - return await getCacheValue(systemCache, key); + return await getCacheValue(systemCache, key, true); } export async function setSchemaCache(schema: SchemaOverview): Promise { @@ -145,10 +145,10 @@ export async function setCacheValue( await cache.set(key, compressed, ttl); } -export async function getCacheValue(cache: Keyv, key: string): Promise { +export async function getCacheValue(cache: Keyv, key: string, parse: boolean): Promise { const value = await cache.get(key); if (!value) return undefined; - const decompressed = await decompress(value); + const decompressed = await decompress(value, parse); return decompressed; } diff --git a/api/src/middleware/cache.ts b/api/src/middleware/cache.ts index e289646b5..bb6de7fef 100644 --- a/api/src/middleware/cache.ts +++ b/api/src/middleware/cache.ts @@ -35,7 +35,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) let cacheExpiryDate; try { - cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`))?.exp; + cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`, true))?.exp; } catch (err: any) { logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); @@ -48,7 +48,9 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) res.setHeader('Vary', 'Origin, Cache-Control'); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'HIT'); - return res.json(cachedData); + res.writeHead(200, ['Content-Type', 'application/json']); + res.write(cachedData); + return res.end(); } else { if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); return next(); diff --git a/api/src/utils/get-permissions.ts b/api/src/utils/get-permissions.ts index ff90c85f0..b1268f954 100644 --- a/api/src/utils/get-permissions.ts +++ b/api/src/utils/get-permissions.ts @@ -37,7 +37,8 @@ export async function getPermissions(accountability: Accountability, schema: Sch const cachedFilterContext = await getCacheValue( cache, - `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}` + `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}`, + true ); if (cachedFilterContext) { From 50e31fd1e782b211948932e94f6633f43dc8ad84 Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Mon, 5 Feb 2024 16:40:34 +0100 Subject: [PATCH 3/7] Changest --- .changeset/wicked-laws-fetch.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/wicked-laws-fetch.md diff --git a/.changeset/wicked-laws-fetch.md b/.changeset/wicked-laws-fetch.md new file mode 100644 index 000000000..56e020926 --- /dev/null +++ b/.changeset/wicked-laws-fetch.md @@ -0,0 +1,5 @@ +--- +'@wbce-d9/api': major +--- + +optimization on cache middleware From 104c4c4320b62f9e76f8a2e0fbd93abd8718f2f9 Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Tue, 6 Feb 2024 11:05:44 +0100 Subject: [PATCH 4/7] fix: remove compression, store record, res.json --- api/src/cache.ts | 10 ++++------ api/src/middleware/cache.ts | 6 ++---- api/src/utils/get-permissions.ts | 3 +-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/api/src/cache.ts b/api/src/cache.ts index 6ec7f26fa..b8a5f0e8e 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -110,7 +110,7 @@ export async function setSystemCache(key: string, value: any, ttl?: number): Pro export async function getSystemCache(key: string): Promise> { const { systemCache } = getCache(); - return await getCacheValue(systemCache, key, true); + return await getCacheValue(systemCache, key); } export async function setSchemaCache(schema: SchemaOverview): Promise { @@ -141,15 +141,13 @@ export async function setCacheValue( value: Record | Record[], ttl?: number ) { - const compressed = await compress(value); - await cache.set(key, compressed, ttl); + await cache.set(key, value, ttl); } -export async function getCacheValue(cache: Keyv, key: string, parse: boolean): Promise { +export async function getCacheValue(cache: Keyv, key: string): Promise { const value = await cache.get(key); if (!value) return undefined; - const decompressed = await decompress(value, parse); - return decompressed; + return value; } function getKeyvInstance(store: Store, ttl: number | undefined, namespaceSuffix?: string): Keyv { diff --git a/api/src/middleware/cache.ts b/api/src/middleware/cache.ts index bb6de7fef..e289646b5 100644 --- a/api/src/middleware/cache.ts +++ b/api/src/middleware/cache.ts @@ -35,7 +35,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) let cacheExpiryDate; try { - cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`, true))?.exp; + cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`))?.exp; } catch (err: any) { logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); @@ -48,9 +48,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) res.setHeader('Vary', 'Origin, Cache-Control'); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'HIT'); - res.writeHead(200, ['Content-Type', 'application/json']); - res.write(cachedData); - return res.end(); + return res.json(cachedData); } else { if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); return next(); diff --git a/api/src/utils/get-permissions.ts b/api/src/utils/get-permissions.ts index b1268f954..ff90c85f0 100644 --- a/api/src/utils/get-permissions.ts +++ b/api/src/utils/get-permissions.ts @@ -37,8 +37,7 @@ export async function getPermissions(accountability: Accountability, schema: Sch const cachedFilterContext = await getCacheValue( cache, - `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}`, - true + `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}` ); if (cachedFilterContext) { From 93e80f4bed2f16da903b0f921d4f0e3c462e1647 Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Tue, 6 Feb 2024 11:11:29 +0100 Subject: [PATCH 5/7] fix: remove import --- api/src/cache.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/cache.ts b/api/src/cache.ts index b8a5f0e8e..4c313bcdb 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -5,7 +5,6 @@ import Keyv from 'keyv'; import env from './env.js'; import logger from './logger.js'; import { getMessenger } from './messenger.js'; -import { compress, decompress } from './utils/compress.js'; import { getConfigFromEnv } from './utils/get-config-from-env.js'; import { getMilliseconds } from './utils/get-milliseconds.js'; import { validateEnv } from './utils/validate-env.js'; From 15ec9bde0359b781b3942c9c8971d3e939fc84a9 Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Tue, 6 Feb 2024 11:05:44 +0100 Subject: [PATCH 6/7] fix: remove compression, store record, res.json --- api/src/cache.ts | 10 ++++------ api/src/middleware/cache.ts | 6 ++---- api/src/utils/get-permissions.ts | 3 +-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/api/src/cache.ts b/api/src/cache.ts index 6ec7f26fa..b8a5f0e8e 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -110,7 +110,7 @@ export async function setSystemCache(key: string, value: any, ttl?: number): Pro export async function getSystemCache(key: string): Promise> { const { systemCache } = getCache(); - return await getCacheValue(systemCache, key, true); + return await getCacheValue(systemCache, key); } export async function setSchemaCache(schema: SchemaOverview): Promise { @@ -141,15 +141,13 @@ export async function setCacheValue( value: Record | Record[], ttl?: number ) { - const compressed = await compress(value); - await cache.set(key, compressed, ttl); + await cache.set(key, value, ttl); } -export async function getCacheValue(cache: Keyv, key: string, parse: boolean): Promise { +export async function getCacheValue(cache: Keyv, key: string): Promise { const value = await cache.get(key); if (!value) return undefined; - const decompressed = await decompress(value, parse); - return decompressed; + return value; } function getKeyvInstance(store: Store, ttl: number | undefined, namespaceSuffix?: string): Keyv { diff --git a/api/src/middleware/cache.ts b/api/src/middleware/cache.ts index bb6de7fef..e289646b5 100644 --- a/api/src/middleware/cache.ts +++ b/api/src/middleware/cache.ts @@ -35,7 +35,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) let cacheExpiryDate; try { - cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`, true))?.exp; + cacheExpiryDate = (await getCacheValue(cache, `${key}__expires_at`))?.exp; } catch (err: any) { logger.warn(err, `[cache] Couldn't read key ${`${key}__expires_at`}. ${err.message}`); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); @@ -48,9 +48,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next) res.setHeader('Vary', 'Origin, Cache-Control'); if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'HIT'); - res.writeHead(200, ['Content-Type', 'application/json']); - res.write(cachedData); - return res.end(); + return res.json(cachedData); } else { if (env['CACHE_STATUS_HEADER']) res.setHeader(`${env['CACHE_STATUS_HEADER']}`, 'MISS'); return next(); diff --git a/api/src/utils/get-permissions.ts b/api/src/utils/get-permissions.ts index b1268f954..ff90c85f0 100644 --- a/api/src/utils/get-permissions.ts +++ b/api/src/utils/get-permissions.ts @@ -37,8 +37,7 @@ export async function getPermissions(accountability: Accountability, schema: Sch const cachedFilterContext = await getCacheValue( cache, - `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}`, - true + `filterContext-${hash({ user, role, permissions: cachedPermissions['permissions'] })}` ); if (cachedFilterContext) { From afe5a771cab9e6cf95ddabfc7711864d647a2af9 Mon Sep 17 00:00:00 2001 From: WBCE_prod Date: Tue, 6 Feb 2024 11:11:29 +0100 Subject: [PATCH 7/7] fix: remove import --- api/src/cache.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/cache.ts b/api/src/cache.ts index b8a5f0e8e..4c313bcdb 100644 --- a/api/src/cache.ts +++ b/api/src/cache.ts @@ -5,7 +5,6 @@ import Keyv from 'keyv'; import env from './env.js'; import logger from './logger.js'; import { getMessenger } from './messenger.js'; -import { compress, decompress } from './utils/compress.js'; import { getConfigFromEnv } from './utils/get-config-from-env.js'; import { getMilliseconds } from './utils/get-milliseconds.js'; import { validateEnv } from './utils/validate-env.js';