From 225ca5021f3eade82fb4c22a894a404c0f888afc Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Thu, 14 Nov 2024 16:03:57 +0100 Subject: [PATCH 01/10] Fix. --- lamar/tasks/feature_extraction.py | 14 ++++++++++++++ lamar/tasks/feature_matching.py | 9 +++++++++ lamar/tasks/pose_estimation.py | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lamar/tasks/feature_extraction.py b/lamar/tasks/feature_extraction.py index 82320aa..9a2f67a 100644 --- a/lamar/tasks/feature_extraction.py +++ b/lamar/tasks/feature_extraction.py @@ -147,4 +147,18 @@ class RetrievalFeatureExtraction(FeatureExtraction): 'preprocessing': {'resize_max': 640}, } }, + 'cosplace': { + 'name': 'cosplace', + 'hloc': { + 'model': {'name': 'cosplace'}, + 'preprocessing': {'resize_max': 640}, + } + }, + 'openibl': { + 'name': 'openibl', + 'hloc': { + 'model': {'name': 'openibl'}, + 'preprocessing': {'resize_max': 640}, + } + } } diff --git a/lamar/tasks/feature_matching.py b/lamar/tasks/feature_matching.py index 5a36347..bfba8c1 100644 --- a/lamar/tasks/feature_matching.py +++ b/lamar/tasks/feature_matching.py @@ -33,6 +33,15 @@ class FeatureMatching: }, }, }, + 'lightglue': { + 'name': 'lightglue', + 'hloc': { + 'model': { + 'name': 'lightglue', + 'features': 'superpoint', + }, + }, + }, 'mnn': { 'name': 'mnn', 'hloc': { diff --git a/lamar/tasks/pose_estimation.py b/lamar/tasks/pose_estimation.py index 585136a..3a42300 100644 --- a/lamar/tasks/pose_estimation.py +++ b/lamar/tasks/pose_estimation.py @@ -51,7 +51,7 @@ class PoseEstimation: method2class = {} method = None evaluation = { - 'Rt_thresholds': [(1, 0.1), (5, 1.)], + 'Rt_thresholds': [(5, .5), (10, .25), (10, .5), (10, 1), (10, 2.5), (10, 5), (10, 10)], } def __init_subclass__(cls): From d2346742c35f40aa01c83b27b8558cd635ef9775 Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Thu, 14 Nov 2024 16:04:01 +0100 Subject: [PATCH 02/10] Other. --- scantools/proc/overlap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scantools/proc/overlap.py b/scantools/proc/overlap.py index afffd5f..d4dd780 100644 --- a/scantools/proc/overlap.py +++ b/scantools/proc/overlap.py @@ -100,7 +100,7 @@ def trajectory_overlap(self, keys_q: List, session_q: Session, if is_self: keys_r, session_r, poses_r = keys_q, session_q, poses_q - overlap_matrix = np.full((len(keys_q), len(keys_r)), -1, float) + overlap_matrix = np.full((len(keys_q), len(keys_r)), -1, dtype=np.float16) overlap_matrix[discard] = 0 # cache the image poses as they might be compositions of rig poses @@ -195,8 +195,8 @@ def compute_overlaps_for_sequence(capture: Capture, id_q: str, id_ref: str, valid_image_indices_list = [np.array(list(range(len(keys_ref))))] selected_keys_ref_list = [keys_ref] - ov_q2r = np.zeros([len(keys_q), len(keys_ref)]) - ov_r2q = np.zeros([len(keys_ref), len(keys_q)]) + ov_q2r = np.zeros([len(keys_q), len(keys_ref)], dtype=np.float16) + ov_r2q = np.zeros([len(keys_ref), len(keys_q)], dtype=np.float16) for sub_info in zip(sub_mesh_id_list, valid_image_indices_list, selected_keys_ref_list): sub_mesh_id, valid_image_indices, selected_keys_ref = sub_info sub_mesh_path = capture.proc_path(id_ref) / session_ref.proc.meshes[sub_mesh_id] From 8cad2d98350acec9686cccc1bc6d039fc0435752 Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sat, 25 Jan 2025 14:27:34 +0100 Subject: [PATCH 03/10] dbg. --- lamar/tasks/feature_extraction.py | 19 +++++++++++++++++++ lamar/tasks/feature_matching.py | 9 +++++++++ 2 files changed, 28 insertions(+) diff --git a/lamar/tasks/feature_extraction.py b/lamar/tasks/feature_extraction.py index 9a2f67a..ceb8467 100644 --- a/lamar/tasks/feature_extraction.py +++ b/lamar/tasks/feature_extraction.py @@ -34,6 +34,18 @@ class FeatureExtraction: }, }, }, + 'dumbpoint': { + 'name': 'dumbpoint', + 'hloc': { + 'model': { + 'name': 'dumbpoint', + }, + 'preprocessing': { + 'grayscale': True, + 'resize_max': 1024, + }, + }, + }, 'r2d2': { 'name': 'r2d2', 'hloc': { @@ -160,5 +172,12 @@ class RetrievalFeatureExtraction(FeatureExtraction): 'model': {'name': 'openibl'}, 'preprocessing': {'resize_max': 640}, } + }, + 'salad': { + 'name': 'salad', + 'hloc': { + 'model': {'name': 'salad'}, + 'preprocessing': {'resize_max': 640}, + } } } diff --git a/lamar/tasks/feature_matching.py b/lamar/tasks/feature_matching.py index bfba8c1..5da6e7a 100644 --- a/lamar/tasks/feature_matching.py +++ b/lamar/tasks/feature_matching.py @@ -42,6 +42,15 @@ class FeatureMatching: }, }, }, + 'loftr': { + 'name': 'loftr', + 'hloc': { + 'model': { + 'name': 'loftr', + 'features': 'dumbpoint', + }, + }, + }, 'mnn': { 'name': 'mnn', 'hloc': { From 03d9748b9287a3dee63009e49783b4b57680be0d Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Tue, 28 Jan 2025 14:42:43 +0100 Subject: [PATCH 04/10] Fix. --- lamar/tasks/mapping.py | 22 +++++++++++++++------- lamar/tasks/pose_estimation.py | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index 1406bbe..4cbb95d 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -94,6 +94,7 @@ def __init__(self, config, outputs, capture, session_id, self.name2key[image.name]: image.image_id for image in self.reconstruction.images.values() } + self.points3d_cache = {} def run(self, capture): run_capture_to_empty_colmap.run(capture, [self.session_id], self.paths.sfm_empty) @@ -111,14 +112,21 @@ def get_points3D(self, key, point2D_indices): valid = [] xyz = [] ids = [] - if len(image.points2D) > 0: - for idx in point2D_indices: - p = image.points2D[idx] - valid.append(p.has_point3D()) - if valid[-1]: - ids.append(p.point3D_id) + if key not in self.points3d_cache: + ids = [] + xyz = [] + for p2d in image.points2D: + if p2d.has_point3D(): + ids.append(p2d.point3D_id) xyz.append(self.reconstruction.points3D[ids[-1]].xyz) - return np.array(valid, bool), xyz, ids + else: + ids.append(-1) + xyz.append([np.nan, np.nan, np.nan]) + self.points3d_cache[key] = (np.array(ids), np.array(xyz)) + ids, xyz = self.points3d_cache[key] + valid = ids[point2D_indices] != -1 + return valid, xyz[point2D_indices][valid], ids[point2D_indices][valid] + class MeshLifting(Mapping): diff --git a/lamar/tasks/pose_estimation.py b/lamar/tasks/pose_estimation.py index 3a42300..d8f3708 100644 --- a/lamar/tasks/pose_estimation.py +++ b/lamar/tasks/pose_estimation.py @@ -71,7 +71,7 @@ def __init__(self, config, outputs, capture, query_id, matching: FeatureMatching, mapping: Mapping, query_keys: list = None, - parallel: bool = True, + parallel: bool = False, return_covariance: bool = False, override_workdir_root: Path = None): if extraction.config['name'] != mapping.extraction.config['name']: From b69e7bb4f126e798d74058c3412f14ce4b794aef Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Tue, 28 Jan 2025 14:55:36 +0100 Subject: [PATCH 05/10] Fix. --- lamar/tasks/mapping.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index 4cbb95d..8ece887 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -108,11 +108,8 @@ def run(self, capture): ) def get_points3D(self, key, point2D_indices): - image = self.reconstruction.images[self.key2imageid[key]] - valid = [] - xyz = [] - ids = [] if key not in self.points3d_cache: + image = self.reconstruction.images[self.key2imageid[key]] ids = [] xyz = [] for p2d in image.points2D: @@ -124,6 +121,9 @@ def get_points3D(self, key, point2D_indices): xyz.append([np.nan, np.nan, np.nan]) self.points3d_cache[key] = (np.array(ids), np.array(xyz)) ids, xyz = self.points3d_cache[key] + if len(ids) == 0: + # Not registered. + return np.array([], bool), [], [] valid = ids[point2D_indices] != -1 return valid, xyz[point2D_indices][valid], ids[point2D_indices][valid] From fc930ec3173b2984d15a0fd1aa52f4de55e53a5e Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sun, 23 Mar 2025 10:28:14 +0100 Subject: [PATCH 06/10] Some reverts. --- lamar/tasks/feature_extraction.py | 12 ------------ lamar/tasks/feature_matching.py | 9 --------- lamar/tasks/mapping.py | 1 + lamar/tasks/pose_estimation.py | 4 ++-- 4 files changed, 3 insertions(+), 23 deletions(-) diff --git a/lamar/tasks/feature_extraction.py b/lamar/tasks/feature_extraction.py index ceb8467..6215076 100644 --- a/lamar/tasks/feature_extraction.py +++ b/lamar/tasks/feature_extraction.py @@ -34,18 +34,6 @@ class FeatureExtraction: }, }, }, - 'dumbpoint': { - 'name': 'dumbpoint', - 'hloc': { - 'model': { - 'name': 'dumbpoint', - }, - 'preprocessing': { - 'grayscale': True, - 'resize_max': 1024, - }, - }, - }, 'r2d2': { 'name': 'r2d2', 'hloc': { diff --git a/lamar/tasks/feature_matching.py b/lamar/tasks/feature_matching.py index 5da6e7a..bfba8c1 100644 --- a/lamar/tasks/feature_matching.py +++ b/lamar/tasks/feature_matching.py @@ -42,15 +42,6 @@ class FeatureMatching: }, }, }, - 'loftr': { - 'name': 'loftr', - 'hloc': { - 'model': { - 'name': 'loftr', - 'features': 'dumbpoint', - }, - }, - }, 'mnn': { 'name': 'mnn', 'hloc': { diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index 8ece887..bd87644 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -108,6 +108,7 @@ def run(self, capture): ) def get_points3D(self, key, point2D_indices): + # TODO(WIP): This cache should have a size limit, maybe LRU. if key not in self.points3d_cache: image = self.reconstruction.images[self.key2imageid[key]] ids = [] diff --git a/lamar/tasks/pose_estimation.py b/lamar/tasks/pose_estimation.py index d8f3708..8a110d3 100644 --- a/lamar/tasks/pose_estimation.py +++ b/lamar/tasks/pose_estimation.py @@ -51,7 +51,7 @@ class PoseEstimation: method2class = {} method = None evaluation = { - 'Rt_thresholds': [(5, .5), (10, .25), (10, .5), (10, 1), (10, 2.5), (10, 5), (10, 10)], + 'Rt_thresholds': [(1, .1), (5, .5), (5, 1), (10, .25), (10, .5), (10, 1), (10, 2.5), (10, 5), (10, 10)], } def __init_subclass__(cls): @@ -71,7 +71,7 @@ def __init__(self, config, outputs, capture, query_id, matching: FeatureMatching, mapping: Mapping, query_keys: list = None, - parallel: bool = False, + parallel: bool = True, return_covariance: bool = False, override_workdir_root: Path = None): if extraction.config['name'] != mapping.extraction.config['name']: From 55b567a9a1b8aadb94c0dfe58f26f93a57a26432 Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sun, 23 Mar 2025 16:35:13 +0100 Subject: [PATCH 07/10] No parallel. --- lamar/tasks/pose_estimation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lamar/tasks/pose_estimation.py b/lamar/tasks/pose_estimation.py index 8a110d3..aba8a65 100644 --- a/lamar/tasks/pose_estimation.py +++ b/lamar/tasks/pose_estimation.py @@ -71,7 +71,7 @@ def __init__(self, config, outputs, capture, query_id, matching: FeatureMatching, mapping: Mapping, query_keys: list = None, - parallel: bool = True, + parallel: bool = False, return_covariance: bool = False, override_workdir_root: Path = None): if extraction.config['name'] != mapping.extraction.config['name']: From 0325cd746403799d1699478ff51c7f2d63e323ee Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sun, 23 Mar 2025 16:52:03 +0100 Subject: [PATCH 08/10] Fix. --- lamar/tasks/mapping.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index bd87644..cb58c03 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -14,6 +14,7 @@ from .feature_matching import FeatureMatching from ..utils.capture import list_images_for_session from ..utils.misc import same_configs, write_config +from ..utils.lru_cache import PerKeyLockLRUCache logger = logging.getLogger(__name__) @@ -94,6 +95,10 @@ def __init__(self, config, outputs, capture, session_id, self.name2key[image.name]: image.image_id for image in self.reconstruction.images.values() } + # We cache the 3D points for mapping images to avoid parsing them from + # the reconstruction each time. If we count 5K keypoints per image, this + # leads to ~80KB / image, so even for 50K mapping images, this would + # only be ~4GB of RAM. self.points3d_cache = {} def run(self, capture): @@ -108,7 +113,7 @@ def run(self, capture): ) def get_points3D(self, key, point2D_indices): - # TODO(WIP): This cache should have a size limit, maybe LRU. + # TODO(WIP): This is not thread safe... if key not in self.points3d_cache: image = self.reconstruction.images[self.key2imageid[key]] ids = [] From 8efac4f1d5b0ffff8f54dd3869aca42cbda8d1ab Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sun, 23 Mar 2025 16:55:28 +0100 Subject: [PATCH 09/10] Fix. --- lamar/tasks/mapping.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index cb58c03..b8d4ac8 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -14,7 +14,6 @@ from .feature_matching import FeatureMatching from ..utils.capture import list_images_for_session from ..utils.misc import same_configs, write_config -from ..utils.lru_cache import PerKeyLockLRUCache logger = logging.getLogger(__name__) From 938a556bfb5c285a0d3c1a61d4c50ac5b88bd48f Mon Sep 17 00:00:00 2001 From: Mihai Dusmanu Date: Sun, 23 Mar 2025 17:58:04 +0100 Subject: [PATCH 10/10] Thread safe. --- lamar/tasks/mapping.py | 51 +++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/lamar/tasks/mapping.py b/lamar/tasks/mapping.py index b8d4ac8..b3ff3cb 100644 --- a/lamar/tasks/mapping.py +++ b/lamar/tasks/mapping.py @@ -1,6 +1,7 @@ import logging from pathlib import Path from copy import deepcopy +from threading import Lock import numpy as np import pycolmap @@ -96,9 +97,17 @@ def __init__(self, config, outputs, capture, session_id, } # We cache the 3D points for mapping images to avoid parsing them from # the reconstruction each time. If we count 5K keypoints per image, this - # leads to ~80KB / image, so even for 50K mapping images, this would - # only be ~4GB of RAM. + # leads to ~160KB / image, so even for 25K mapping images, this would + # only be ~4GB of RAM. For very large datasets, we might want to look + # into a LRU cache. self.points3d_cache = {} + # We use a lock per image to make sure other threads using different + # images can run in parallel. These locks block for the expensive + # reconstruction parsing. + self.image_locks = {} + # We use a lock for the image locks to make sure that we do not + # create multiple locks for the same image. + self.lock = Lock() def run(self, capture): run_capture_to_empty_colmap.run(capture, [self.session_id], self.paths.sfm_empty) @@ -111,20 +120,7 @@ def run(self, capture): self.matching.paths.matches, ) - def get_points3D(self, key, point2D_indices): - # TODO(WIP): This is not thread safe... - if key not in self.points3d_cache: - image = self.reconstruction.images[self.key2imageid[key]] - ids = [] - xyz = [] - for p2d in image.points2D: - if p2d.has_point3D(): - ids.append(p2d.point3D_id) - xyz.append(self.reconstruction.points3D[ids[-1]].xyz) - else: - ids.append(-1) - xyz.append([np.nan, np.nan, np.nan]) - self.points3d_cache[key] = (np.array(ids), np.array(xyz)) + def _get_points3D_from_cache(self, key, point2D_indices): ids, xyz = self.points3d_cache[key] if len(ids) == 0: # Not registered. @@ -133,6 +129,29 @@ def get_points3D(self, key, point2D_indices): return valid, xyz[point2D_indices][valid], ids[point2D_indices][valid] + def get_points3D(self, key, point2D_indices): + if key not in self.image_locks: + with self.lock: + # Key might have been added while we were waiting for the lock. + if key not in self.image_locks: + self.image_locks[key] = Lock() + if key not in self.points3d_cache: + with self.image_locks[key]: + # Key might have been added while we were waiting for the lock. + if key not in self.points3d_cache: + image = self.reconstruction.images[self.key2imageid[key]] + ids = [] + xyz = [] + for p2d in image.points2D: + if p2d.has_point3D(): + ids.append(p2d.point3D_id) + xyz.append(self.reconstruction.points3D[ids[-1]].xyz) + else: + ids.append(-1) + xyz.append([np.nan, np.nan, np.nan]) + self.points3d_cache[key] = (np.array(ids), np.array(xyz)) + return self._get_points3D_from_cache(key, point2D_indices) + class MeshLifting(Mapping): method = {