From c6726fea6684125a57a1b356777b4c26d5303c35 Mon Sep 17 00:00:00 2001 From: gmaillet Date: Mon, 1 Jul 2024 14:41:30 +0200 Subject: [PATCH 1/6] add advisory lock to avoid concurrent branch modifications --- db/db.js | 20 ++++++++++++++++++++ middlewares/branch.js | 38 ++++++++++++++++++++++++++++++++++++++ routes/branch.js | 1 + routes/graph.js | 1 + routes/patch.js | 5 +++++ routes/wmts.js | 1 + 6 files changed, 66 insertions(+) diff --git a/db/db.js b/db/db.js index 77c7eb862..fe0ce1a18 100644 --- a/db/db.js +++ b/db/db.js @@ -83,6 +83,24 @@ async function getCachePath(pgClient, idBranch) { throw new Error('idBranch non valide'); } +async function lockSharedBranch(pgClient, idBranch) { + debug(` ~~lockSharedBranch (idBranch: ${idBranch})`); + await pgClient.query( + 'SELECT pg_advisory_xact_lock_shared($1)', + [idBranch], + ); + debug(` ~~lockSharedBranch (idBranch: ${idBranch}) done`); +} + +async function lockBranch(pgClient, idBranch) { + debug(` ~~lockBranch (idBranch: ${idBranch})`); + await pgClient.query( + 'SELECT pg_advisory_xact_lock($1)', + [idBranch], + ); + debug(` ~~lockBranch (idBranch: ${idBranch}) done`); +} + async function getBranches(pgClient, idCache) { debug(` ~~getBranches (idCache: ${idCache})`); let results; @@ -546,6 +564,8 @@ module.exports = { deleteCache, insertListOpi, getCachePath, + lockSharedBranch, + lockBranch, getBranches, // getIdCacheFromPath, insertBranch, diff --git a/middlewares/branch.js b/middlewares/branch.js index 033b06ea5..502e955f8 100644 --- a/middlewares/branch.js +++ b/middlewares/branch.js @@ -269,10 +269,48 @@ async function getCachePath(req, _res, next) { next(); } +async function lockShared(req, _res, next) { + debug('>>GET lockShared'); + if (req.error) { + next(); + return; + } + const params = matchedData(req); + const { idBranch } = params; + try { + await db.lockSharedBranch(req.client, idBranch); + } catch (error) { + debug(error); + req.error = error; + } + debug(' next>>'); + next(); +} + +async function lock(req, _res, next) { + debug('>>GET lock'); + if (req.error) { + next(); + return; + } + const params = matchedData(req); + const { idBranch } = params; + try { + await db.lockBranch(req.client, idBranch); + } catch (error) { + debug(error); + req.error = error; + } + debug(' next>>'); + next(); +} + module.exports = { getBranches, postBranch, deleteBranch, rebase, getCachePath, + lockShared, + lock, }; diff --git a/routes/branch.js b/routes/branch.js index 728252121..73798a0e7 100644 --- a/routes/branch.js +++ b/routes/branch.js @@ -49,6 +49,7 @@ router.delete('/branch', .withMessage(createErrMsg.invalidParameter('idBranch')), ], validateParams, + branch.lock, branch.deleteBranch, pgClient.close, returnMsg); diff --git a/routes/graph.js b/routes/graph.js index 07ac36642..6f05aba75 100644 --- a/routes/graph.js +++ b/routes/graph.js @@ -29,6 +29,7 @@ router.get('/:idBranch/graph', validateParams, branch.getCachePath, cache.getOverviews, + branch.lockShared, graph.getGraph, pgClient.close, returnMsg); diff --git a/routes/patch.js b/routes/patch.js index eb9ded37b..b7c2941f9 100644 --- a/routes/patch.js +++ b/routes/patch.js @@ -61,6 +61,7 @@ router.get('/:idBranch/patches', .withMessage(createErrMsg.invalidParameter('idBranch')), ], validateParams, + branch.lockShared, patch.getPatches, pgClient.close, returnMsg); @@ -79,6 +80,7 @@ router.post('/:idBranch/patch', validateParams, branch.getCachePath, cache.getOverviews, + branch.lock, patch.postPatch, pgClient.close, returnMsg); @@ -95,6 +97,7 @@ router.put('/:idBranch/patch/undo', validateParams, branch.getCachePath, cache.getOverviews, + branch.lock, patch.undo, pgClient.close, returnMsg); @@ -111,6 +114,7 @@ router.put('/:idBranch/patch/redo', validateParams, branch.getCachePath, cache.getOverviews, + branch.lock, patch.redo, pgClient.close, returnMsg); @@ -127,6 +131,7 @@ router.put('/:idBranch/patches/clear', validateParams, branch.getCachePath, cache.getOverviews, + branch.lock, patch.clear, pgClient.close, returnMsg); diff --git a/routes/wmts.js b/routes/wmts.js index 5f1e6cac0..ebbfafa92 100644 --- a/routes/wmts.js +++ b/routes/wmts.js @@ -69,6 +69,7 @@ router.get('/:idBranch/wmts', validateParams, branch.getCachePath, cache.getOverviews, + branch.lockShared, [ query('TILEMATRIXSET').if(query('REQUEST').isIn(['GetTile', 'GetFeatureInfo'])) .exists().withMessage(createErrMsg.missingParameter('TILEMATRIXSET')) From 9497b203f7c153e6ac9fad62074b41275f5380a6 Mon Sep 17 00:00:00 2001 From: gmaillet Date: Wed, 24 Jul 2024 17:25:47 +0200 Subject: [PATCH 2/6] check all files before undo/redo process --- middlewares/patch.js | 220 +++++++++++++++++++++++++++++-------------- 1 file changed, 147 insertions(+), 73 deletions(-) diff --git a/middlewares/patch.js b/middlewares/patch.js index 590b6ec7d..4453e9300 100644 --- a/middlewares/patch.js +++ b/middlewares/patch.js @@ -421,6 +421,7 @@ async function undo(req, _res, next) { // pour chaque tuile, trouver le numéro de version le plus élevé inférieur au numéro de patch const errors = []; const histories = []; + const toRenamed = []; // Object.values(slabs).forEach((slab, indexSlab) => { // slabs.forEach((slab, indexSlab) => { slabs.forEach((slab, indexSlab) => { @@ -436,9 +437,7 @@ async function undo(req, _res, next) { debug("erreur d'historique"); errors.push(`error: history on tile ${cogPath}`); debug('erreur : ', history, lastPatchNum); - // res.status(404).send(`erreur d'historique sur la tuile ${cogPath}`); } else { - // histories[indexSlab] = history; histories[indexSlab] = history; } }); @@ -451,15 +450,81 @@ async function undo(req, _res, next) { next(); return; } - // Object.values(slabs).forEach((slab, indexSlab) => { + // Premiere boucle: on s'assure que tous les fichiers sont dispo avant de commencer slabs.forEach((slab, indexSlab) => { const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); - const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); - const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); // on récupère la version à restaurer const history = histories[indexSlab]; const patchIdPrev = history[history.length - 1]; const idSelected = history[history.length - 2]; + // debug(' version selectionnée pour la tuile :', idSelected); + const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); + const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); + + debug(` dalle ${slab.z}/${slab.y}/${slab.x} : version ${idSelected} selectionnée`); + const todo = []; + // renommer les images pour pointer sur ce numéro de version + todo.push([ + path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`), + path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`), + ]); + if (withRgb) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`), + ]); + } + if (withIr) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}i.tif`), + ]); + } + if (idSelected !== 'orig') { + todo.push([ + path.join(graphDir, `${idBranch}_${cogPath.filename}_${idSelected}.tif`), + path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`), + ]); + if (withRgb) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${idSelected}.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`), + ]); + } + if (withIr) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${idSelected}i.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`), + ]); + } + } + // on teste que tous les fichiers sont accessibles + try { + for (let i = 0; i < todo.length; i += 1) { + /* eslint-disable-next-line no-bitwise */ + fs.accessSync(todo[i][0], fs.constants.R_OK | fs.constants.W_OK); + } + } catch (e) { + errors.push(`error: fileaccess ${e}`); + } + toRenamed[indexSlab] = todo; + }); + if (errors.length > 0) { + req.error = { + msg: errors, + code: 404, + function: 'undo', + }; + next(); + return; + } + // Deuxieme boucle: tout est ok, on peut commencer les modifications sur disque + slabs.forEach((slab, indexSlab) => { + const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); + const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); + const history = histories[indexSlab]; + const todo = toRenamed[indexSlab]; + const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); // mise à jour de l'historique let newHistory = ''; for (let i = 0; i < (history.length - 1); i += 1) { @@ -468,49 +533,15 @@ async function undo(req, _res, next) { } debug('newHistory : ', newHistory); fs.writeFileSync(`${urlHistory}`, newHistory); - debug(` dalle ${slab.z}/${slab.y}/${slab.x} : version ${idSelected} selectionnée`); - // debug(' version selectionnée pour la tuile :', idSelected); - const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); - const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); - // renommer les images pour pointer sur ce numéro de version - const urlGraph = path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`); - const urlOrthoRgb = path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`); - const urlOrthoIr = path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`); - const urlGraphSelected = path.join(graphDir, `${idBranch}_${cogPath.filename}_${idSelected}.tif`); - const urlOrthoRgbSelected = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${idSelected}.tif`); - const urlOrthoIrSelected = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${idSelected}i.tif`); - - // on renomme les anciennes images - const urlGraphPrev = path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`); - const urlOrthoRgbPrev = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`); - const urlOrthoIrPrev = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}i.tif`); - - rename(urlGraph, urlGraphPrev); - if (withRgb) rename(urlOrthoRgb, urlOrthoRgbPrev); - if (withIr) rename(urlOrthoIr, urlOrthoIrPrev); - - // on renomme les nouvelles images sauf si c'est la version orig - if (idSelected !== 'orig') { - rename(urlGraphSelected, urlGraph); - if (withRgb) rename(urlOrthoRgbSelected, urlOrthoRgb); - if (withIr) rename(urlOrthoIrSelected, urlOrthoIr); + for (let i = 0; i < todo.length; i += 1) { + rename(todo[i][0], todo[i][1]); } }); const result = await db.deactivatePatch(req.client, lastPatchId); debug(result.rowCount); - - // req.selectedBranch.unactivePatches.features = req.selectedBranch.unactivePatches.features - // .concat( - // features, - // ); - // fs.writeFileSync(path.join(req.dir_cache, 'branches.json'), - // JSON.stringify(req.app.branches, null, 4)); - debug('fin du undo'); - // debug('features in activePatches:', activePatches.features.length); - // debug('features in unactivePatches:', req.selectedBranch.unactivePatches.features.length); req.result = { json: `undo: patch ${lastPatchNum} annulé`, code: 200 }; debug(' next>>'); next(); @@ -571,54 +602,97 @@ async function redo(req, _res, next) { // debug(Object.keys(slabs).length, ' dalles impactées'); debug(slabs.length, 'dalles impactées'); - // pour chaque tuile, renommer les images - // Object.values(slabs).forEach((slab) => { - slabs.forEach((slab) => { + + const errors = []; + const histories = []; + const toRenamed = []; + // Premiere boucle: on s'assure que tous les fichiers sont dispo avant de commencer + slabs.forEach((slab, indexSlab) => { debug(slab); const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); debug(cogPath); const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); - // on met a jour l'historique const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); const history = `${fs.readFileSync(`${urlHistory}`)};${patchNumRedo}`; const tabHistory = history.split(';'); const patchIdPrev = tabHistory[tabHistory.length - 2]; - fs.writeFileSync(`${urlHistory}`, history); - // on verifie si la tuile a été effectivement modifiée par ce patch - const urlGraphSelected = path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}.tif`); - const urlOrthoRgbSelected = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}.tif`); - const urlOrthoIrSelected = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}i.tif`); - // renommer les images pour pointer sur ce numéro de version - const urlGraph = path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`); - const urlOrthoRgb = path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`); - const urlOrthoIr = path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`); - // on renomme les anciennes images - const urlGraphPrev = path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`); - const urlOrthoRgbPrev = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`); - const urlOrthoIrPrev = path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}i.tif`); + histories[indexSlab] = history; + const todo = []; + // on backup la version en cours (si ça n'est pas la orig) if (patchIdPrev !== 'orig') { - rename(urlGraph, urlGraphPrev); - if (withRgb) rename(urlOrthoRgb, urlOrthoRgbPrev); - if (withIr) rename(urlOrthoIr, urlOrthoIrPrev); + todo.push([ + path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`), + path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`), + ]); + if (withRgb) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}.tif`), + ]); + } + if (withIr) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchIdPrev}i.tif`), + ]); + } } - // on renomme les nouvelles images - rename(urlGraphSelected, urlGraph); - if (withRgb) rename(urlOrthoRgbSelected, urlOrthoRgb); - if (withIr) rename(urlOrthoIrSelected, urlOrthoIr); + // on applique la version du patch + todo.push([ + path.join(graphDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}.tif`), + path.join(graphDir, `${idBranch}_${cogPath.filename}.tif`), + ]); + if (withRgb) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}.tif`), + ]); + } + if (withIr) { + todo.push([ + path.join(orthoDir, `${idBranch}_${cogPath.filename}_${patchNumRedo}i.tif`), + path.join(orthoDir, `${idBranch}_${cogPath.filename}i.tif`), + ]); + } + + // on teste que tous les fichiers sont accessibles + try { + for (let i = 0; i < todo.length; i += 1) { + /* eslint-disable-next-line no-bitwise */ + fs.accessSync(todo[i][0], fs.constants.R_OK | fs.constants.W_OK); + } + } catch (e) { + errors.push(`error: fileaccess ${e}`); + } + toRenamed[indexSlab] = todo; }); - // on remet les features dans req.app.activePatches.features - // req.selectedBranch.activePatches.features = req.selectedBranch.activePatches.features.concat( - // features, - // ); - // fs.writeFileSync(path.join(global.dir_cache, 'branches.json'), - // JSON.stringify(req.app.branches, null, 4)); + if (errors.length > 0) { + req.error = { + msg: errors, + code: 404, + function: 'redo', + }; + next(); + return; + } + // Deuxieme boucle: tout est ok, on peut commencer les modifications sur disque + slabs.forEach((slab, indexSlab) => { + const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); + const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); + const history = histories[indexSlab]; + const todo = toRenamed[indexSlab]; - // debug('features in activePatches:', req.selectedBranch.activePatches.features.length); - // debug('features in unactivePatches:', req.selectedBranch.unactivePatches.features.length); + // on met a jour l'historique + const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); + fs.writeFileSync(`${urlHistory}`, history); + for (let i = 0; i < todo.length; i += 1) { + rename(todo[i][0], todo[i][1]); + } + }); const result = await db.reactivatePatch(req.client, patchIdRedo); debug(result.rowCount); From 8617e2c878a6b104dc6c827975b243ca4e0cf49f Mon Sep 17 00:00:00 2001 From: gmaillet Date: Thu, 25 Jul 2024 15:45:18 +0200 Subject: [PATCH 3/6] add fsyncSync to flush disk operations --- middlewares/patch.js | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/middlewares/patch.js b/middlewares/patch.js index 4453e9300..59891d86d 100644 --- a/middlewares/patch.js +++ b/middlewares/patch.js @@ -1,4 +1,5 @@ const debug = require('debug')('patch'); +const os = require('os'); const fs = require('fs'); const canvas = require('canvas'); const turf = require('@turf/turf'); @@ -422,12 +423,14 @@ async function undo(req, _res, next) { const errors = []; const histories = []; const toRenamed = []; + const directories = []; // Object.values(slabs).forEach((slab, indexSlab) => { // slabs.forEach((slab, indexSlab) => { slabs.forEach((slab, indexSlab) => { debug('slab :', slab, indexSlab); const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); + directories.push(opiDir); // on récupère l'historique de cette tuile const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); @@ -460,6 +463,8 @@ async function undo(req, _res, next) { // debug(' version selectionnée pour la tuile :', idSelected); const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); + directories.push(graphDir); + directories.push(orthoDir); debug(` dalle ${slab.z}/${slab.y}/${slab.x} : version ${idSelected} selectionnée`); const todo = []; @@ -535,8 +540,24 @@ async function undo(req, _res, next) { fs.writeFileSync(`${urlHistory}`, newHistory); for (let i = 0; i < todo.length; i += 1) { rename(todo[i][0], todo[i][1]); + if (os.platform() === 'linux') { + // on flush le fichier destination + // pour le cas ou on utilise plusieurs serveurs + const fd = fs.openSync(todo[i][1]); + fs.fsyncSync(fd); + fs.closeSync(fd); + } } }); + // Derniere boucle, on flush tous les dossiers dans lesquels on a fait des modifs + if (os.platform() === 'linux') { + const uniqueDirectories = [...new Set(directories)]; + uniqueDirectories.forEach((d) => { + const fd = fs.openSync(d); + fs.fsyncSync(fd); + fs.closeSync(fd); + }); + } const result = await db.deactivatePatch(req.client, lastPatchId); @@ -606,6 +627,7 @@ async function redo(req, _res, next) { const errors = []; const histories = []; const toRenamed = []; + const directories = []; // Premiere boucle: on s'assure que tous les fichiers sont dispo avant de commencer slabs.forEach((slab, indexSlab) => { debug(slab); @@ -614,6 +636,9 @@ async function redo(req, _res, next) { const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); + directories.push(opiDir); + directories.push(graphDir); + directories.push(orthoDir); // on met a jour l'historique const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); const history = `${fs.readFileSync(`${urlHistory}`)};${patchNumRedo}`; @@ -691,8 +716,24 @@ async function redo(req, _res, next) { fs.writeFileSync(`${urlHistory}`, history); for (let i = 0; i < todo.length; i += 1) { rename(todo[i][0], todo[i][1]); + if (os.platform() === 'linux') { + // on flush le fichier destination + // pour le cas ou on utilise plusieurs serveurs + const fd = fs.openSync(todo[i][1]); + fs.fsyncSync(fd); + fs.closeSync(fd); + } } }); + // Derniere boucle, on flush tous les dossiers dans lesquels on a fait des modifs + if (os.platform() === 'linux') { + const uniqueDirectories = [...new Set(directories)]; + uniqueDirectories.forEach((d) => { + const fd = fs.openSync(d); + fs.fsyncSync(fd); + fs.closeSync(fd); + }); + } const result = await db.reactivatePatch(req.client, patchIdRedo); debug(result.rowCount); From fb1c8c561e7b7de4f89f59bcbcbe408409fe7402 Mon Sep 17 00:00:00 2001 From: gmaillet Date: Thu, 25 Jul 2024 17:18:57 +0200 Subject: [PATCH 4/6] revert and clean --- middlewares/patch.js | 78 ++------------------------------------------ 1 file changed, 2 insertions(+), 76 deletions(-) diff --git a/middlewares/patch.js b/middlewares/patch.js index 59891d86d..d5ab0f0c2 100644 --- a/middlewares/patch.js +++ b/middlewares/patch.js @@ -1,5 +1,4 @@ const debug = require('debug')('patch'); -const os = require('os'); const fs = require('fs'); const canvas = require('canvas'); const turf = require('@turf/turf'); @@ -398,21 +397,6 @@ async function undo(req, _res, next) { debug(`Patch '${lastPatchNum}' à annuler.`); - // const features = []; - // let index = activePatches.features.length - 1; - // const slabs = {}; - // while (index >= 0) { - // const feature = activePatches.features[index]; - // if (feature.properties.num === lastPatchId) { - // features.push(feature); - // activePatches.features.splice(index, 1); - // feature.properties.slabs.forEach((item) => { - // slabs[`${item.x}_${item.y}_${item.z}`] = item; - // }); - // } - // index -= 1; - // } - const slabs = await db.getSlabs(req.client, lastPatchId); debug(slabs); @@ -423,14 +407,11 @@ async function undo(req, _res, next) { const errors = []; const histories = []; const toRenamed = []; - const directories = []; - // Object.values(slabs).forEach((slab, indexSlab) => { // slabs.forEach((slab, indexSlab) => { slabs.forEach((slab, indexSlab) => { debug('slab :', slab, indexSlab); const cogPath = cog.getSlabPath(slab.x, slab.y, slab.z, overviews.pathDepth); const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); - directories.push(opiDir); // on récupère l'historique de cette tuile const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); @@ -463,8 +444,6 @@ async function undo(req, _res, next) { // debug(' version selectionnée pour la tuile :', idSelected); const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); - directories.push(graphDir); - directories.push(orthoDir); debug(` dalle ${slab.z}/${slab.y}/${slab.x} : version ${idSelected} selectionnée`); const todo = []; @@ -540,28 +519,11 @@ async function undo(req, _res, next) { fs.writeFileSync(`${urlHistory}`, newHistory); for (let i = 0; i < todo.length; i += 1) { rename(todo[i][0], todo[i][1]); - if (os.platform() === 'linux') { - // on flush le fichier destination - // pour le cas ou on utilise plusieurs serveurs - const fd = fs.openSync(todo[i][1]); - fs.fsyncSync(fd); - fs.closeSync(fd); - } } }); - // Derniere boucle, on flush tous les dossiers dans lesquels on a fait des modifs - if (os.platform() === 'linux') { - const uniqueDirectories = [...new Set(directories)]; - uniqueDirectories.forEach((d) => { - const fd = fs.openSync(d); - fs.fsyncSync(fd); - fs.closeSync(fd); - }); - } - const result = await db.deactivatePatch(req.client, lastPatchId); + await db.deactivatePatch(req.client, lastPatchId); - debug(result.rowCount); debug('fin du undo'); req.result = { json: `undo: patch ${lastPatchNum} annulé`, code: 200 }; debug(' next>>'); @@ -604,21 +566,6 @@ async function redo(req, _res, next) { debug(`Patch '${patchNumRedo}' à réappliquer.`); - // const features = []; - // const slabs = {}; - // let index = req.selectedBranch.unactivePatches.features.length - 1; - // while (index >= 0) { - // const feature = req.selectedBranch.unactivePatches.features[index]; - // if (feature.properties.patchId === patchNumRedo) { - // features.push(feature); - // feature.properties.slabs.forEach((item) => { - // slabs[`${item.x}_${item.y}_${item.z}`] = item; - // }); - // req.selectedBranch.unactivePatches.features.splice(index, 1); - // } - // index -= 1; - // } - const slabs = await db.getSlabs(req.client, patchIdRedo); // debug(Object.keys(slabs).length, ' dalles impactées'); @@ -627,7 +574,6 @@ async function redo(req, _res, next) { const errors = []; const histories = []; const toRenamed = []; - const directories = []; // Premiere boucle: on s'assure que tous les fichiers sont dispo avant de commencer slabs.forEach((slab, indexSlab) => { debug(slab); @@ -636,9 +582,6 @@ async function redo(req, _res, next) { const graphDir = path.join(req.dir_cache, 'graph', cogPath.dirPath); const orthoDir = path.join(req.dir_cache, 'ortho', cogPath.dirPath); const opiDir = path.join(req.dir_cache, 'opi', cogPath.dirPath); - directories.push(opiDir); - directories.push(graphDir); - directories.push(orthoDir); // on met a jour l'historique const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); const history = `${fs.readFileSync(`${urlHistory}`)};${patchNumRedo}`; @@ -716,27 +659,10 @@ async function redo(req, _res, next) { fs.writeFileSync(`${urlHistory}`, history); for (let i = 0; i < todo.length; i += 1) { rename(todo[i][0], todo[i][1]); - if (os.platform() === 'linux') { - // on flush le fichier destination - // pour le cas ou on utilise plusieurs serveurs - const fd = fs.openSync(todo[i][1]); - fs.fsyncSync(fd); - fs.closeSync(fd); - } } }); - // Derniere boucle, on flush tous les dossiers dans lesquels on a fait des modifs - if (os.platform() === 'linux') { - const uniqueDirectories = [...new Set(directories)]; - uniqueDirectories.forEach((d) => { - const fd = fs.openSync(d); - fs.fsyncSync(fd); - fs.closeSync(fd); - }); - } - const result = await db.reactivatePatch(req.client, patchIdRedo); - debug(result.rowCount); + await db.reactivatePatch(req.client, patchIdRedo); debug('fin du redo'); req.result = { json: `redo: patch ${patchNumRedo} réappliqué`, code: 200 }; From 6eac1a329eb307c296b79a5db1aea29ef82081e9 Mon Sep 17 00:00:00 2001 From: gmaillet Date: Thu, 25 Jul 2024 17:24:47 +0200 Subject: [PATCH 5/6] protect rename with try/catch --- middlewares/patch.js | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/middlewares/patch.js b/middlewares/patch.js index d5ab0f0c2..386ef0ba2 100644 --- a/middlewares/patch.js +++ b/middlewares/patch.js @@ -517,11 +517,23 @@ async function undo(req, _res, next) { } debug('newHistory : ', newHistory); fs.writeFileSync(`${urlHistory}`, newHistory); - for (let i = 0; i < todo.length; i += 1) { - rename(todo[i][0], todo[i][1]); + try { + for (let i = 0; i < todo.length; i += 1) { + rename(todo[i][0], todo[i][1]); + } + } catch (e) { + errors.push(`error: fileaccess ${e}`); } }); - + if (errors.length > 0) { + req.error = { + msg: errors, + code: 404, + function: 'undo', + }; + next(); + return; + } await db.deactivatePatch(req.client, lastPatchId); debug('fin du undo'); @@ -657,10 +669,23 @@ async function redo(req, _res, next) { // on met a jour l'historique const urlHistory = path.join(opiDir, `${idBranch}_${cogPath.filename}_history.packo`); fs.writeFileSync(`${urlHistory}`, history); - for (let i = 0; i < todo.length; i += 1) { - rename(todo[i][0], todo[i][1]); + try { + for (let i = 0; i < todo.length; i += 1) { + rename(todo[i][0], todo[i][1]); + } + } catch (e) { + errors.push(`error: fileaccess ${e}`); } }); + if (errors.length > 0) { + req.error = { + msg: errors, + code: 404, + function: 'redo', + }; + next(); + return; + } await db.reactivatePatch(req.client, patchIdRedo); From 7c648a76c99055e09459af1916ea3c15e51c04f7 Mon Sep 17 00:00:00 2001 From: gmaillet Date: Thu, 25 Jul 2024 17:36:09 +0200 Subject: [PATCH 6/6] protect rename with try/catch --- middlewares/patch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middlewares/patch.js b/middlewares/patch.js index 386ef0ba2..52a204f14 100644 --- a/middlewares/patch.js +++ b/middlewares/patch.js @@ -522,7 +522,7 @@ async function undo(req, _res, next) { rename(todo[i][0], todo[i][1]); } } catch (e) { - errors.push(`error: fileaccess ${e}`); + errors.push(`error: rename ${e}`); } }); if (errors.length > 0) { @@ -674,7 +674,7 @@ async function redo(req, _res, next) { rename(todo[i][0], todo[i][1]); } } catch (e) { - errors.push(`error: fileaccess ${e}`); + errors.push(`error: rename ${e}`); } }); if (errors.length > 0) {