From fb227a2eb7340a055c707e6e64654b676dccc049 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Mon, 14 Oct 2024 21:13:44 -0700 Subject: [PATCH 01/11] Test with Colmap SIFT Features --- environment_mac.yml | 1 + gtsfm/common/image.py | 2 +- gtsfm/common/sensor_width_database.py | 2 +- gtsfm/configs/correspondence/sift.yaml | 1 + .../detector_descriptor/colmap_sift.py | 58 +++++++++++++++++++ .../detector_descriptor/test_colmap_sift.py | 28 +++++++++ 6 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 gtsfm/frontend/detector_descriptor/colmap_sift.py create mode 100644 tests/frontend/detector_descriptor/test_colmap_sift.py diff --git a/environment_mac.yml b/environment_mac.yml index deb57793a..99b66adfa 100644 --- a/environment_mac.yml +++ b/environment_mac.yml @@ -49,6 +49,7 @@ dependencies: - tabulate - simplejson - pycolmap + - pyparsing==3.0.9 # testing - parameterized - pip: diff --git a/gtsfm/common/image.py b/gtsfm/common/image.py index f7248d112..4a5d1d6c9 100644 --- a/gtsfm/common/image.py +++ b/gtsfm/common/image.py @@ -21,7 +21,7 @@ class Image(NamedTuple): value_array: np.ndarray exif_data: Optional[Dict[str, Any]] = None - sensor_width_db: SensorWidthDatabase = SensorWidthDatabase() + sensor_width_db: SensorWidthDatabase = None #SensorWidthDatabase() file_name: Optional[str] = None mask: Optional[np.ndarray] = None diff --git a/gtsfm/common/sensor_width_database.py b/gtsfm/common/sensor_width_database.py index 04cfa37fd..f6245e1ad 100644 --- a/gtsfm/common/sensor_width_database.py +++ b/gtsfm/common/sensor_width_database.py @@ -6,7 +6,7 @@ """ from pathlib import Path -import pandas as pd +#import pandas as pd ASSETS_ROOT = Path(__file__).resolve().parent.parent / "assets" DEFAULT_SENSOR_DB_PATH = ASSETS_ROOT / "camera_details" / "sensor_database.csv" diff --git a/gtsfm/configs/correspondence/sift.yaml b/gtsfm/configs/correspondence/sift.yaml index 8691035a6..6945e1352 100644 --- a/gtsfm/configs/correspondence/sift.yaml +++ b/gtsfm/configs/correspondence/sift.yaml @@ -6,6 +6,7 @@ CorrespondenceGenerator: detector_descriptor: _target_: gtsfm.frontend.cacher.detector_descriptor_cacher.DetectorDescriptorCacher detector_descriptor_obj: + #_target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor _target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor max_keypoints: 5000 diff --git a/gtsfm/frontend/detector_descriptor/colmap_sift.py b/gtsfm/frontend/detector_descriptor/colmap_sift.py new file mode 100644 index 000000000..2f8795f84 --- /dev/null +++ b/gtsfm/frontend/detector_descriptor/colmap_sift.py @@ -0,0 +1,58 @@ +"""SIFT Detector-Descriptor implementation. + +The detector was proposed in 'Distinctive Image Features from Scale-Invariant Keypoints' and is implemented by wrapping +over OpenCV's API. + +References: +- https://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf +- https://docs.opencv.org/3.4.2/d5/d3c/classcv_1_1xfeatures2d_1_1SIFT.html + +Authors: Ayush Baid +""" +from typing import Tuple + +import numpy as np +import pycolmap + +import gtsfm.utils.features as feature_utils +import gtsfm.utils.images as image_utils +from gtsfm.common.image import Image +from gtsfm.common.keypoints import Keypoints +from gtsfm.frontend.detector_descriptor.detector_descriptor_base import DetectorDescriptorBase + + +class ColmapSIFTDetectorDescriptor(DetectorDescriptorBase): + """SIFT detector-descriptor using OpenCV's implementation.""" + + def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: + """Perform feature detection as well as their description. + + Refer to detect() in DetectorBase and describe() in DescriptorBase for details about the output format. + + Args: + image: the input image. + + Returns: + Detected keypoints, with length N <= max_keypoints. + Corr. descriptors, of shape (N, D) where D is the dimension of each descriptor. + """ + + # Convert to grayscale. + gray_image = image_utils.rgb_to_gray_cv(image) + + # Create OpenCV object every time as the object is not pickle-able. + # TODO (travisdriver): Add GPU support + colmap_obj = pycolmap.Sift() + + # Run the OpenCV code. + features, descriptors = colmap_obj.extract(gray_image.value_array) + + # Convert to GTSFM's keypoints. + # TODO (travisdriver): Add scales and orientations + keypoints = Keypoints(coordinates=features[..., :2]) + + # # Filter features. + # keypoints, selection_idxs = keypoints.get_top_k(self.max_keypoints) + # descriptors = descriptors[selection_idxs] + + return keypoints, descriptors diff --git a/tests/frontend/detector_descriptor/test_colmap_sift.py b/tests/frontend/detector_descriptor/test_colmap_sift.py new file mode 100644 index 000000000..55dbb4296 --- /dev/null +++ b/tests/frontend/detector_descriptor/test_colmap_sift.py @@ -0,0 +1,28 @@ +"""Tests for SIFT detector descriptor + +Authors: Ayush Baid +""" +import unittest + +import tests.frontend.detector_descriptor.test_detector_descriptor_base as test_detector_descriptor_base +from gtsfm.frontend.detector.detector_from_joint_detector_descriptor import DetectorFromDetectorDescriptor +from gtsfm.frontend.detector_descriptor.colmap_sift import ColmapSIFTDetectorDescriptor + + +class TestColmapSIFTDetectorDescriptor(test_detector_descriptor_base.TestDetectorDescriptorBase): + """Main test class for detector-description combination base class in frontend. + + All unit test functions defined in TestDetectorDescriptorBase are run automatically. + """ + + def setUp(self): + """Setup the attributes for the tests.""" + super().setUp() + self.detector_descriptor = ColmapSIFTDetectorDescriptor() + + # explicitly set the detector + self.detector = DetectorFromDetectorDescriptor(self.detector_descriptor) + + +if __name__ == "__main__": + unittest.main() From 6ee4bb02372056f3de752323a084cd281c0402f2 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Mon, 14 Oct 2024 21:38:23 -0700 Subject: [PATCH 02/11] fix pyparsing --- environment_linux_cpuonly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment_linux_cpuonly.yml b/environment_linux_cpuonly.yml index 415c51da9..6a21b640e 100644 --- a/environment_linux_cpuonly.yml +++ b/environment_linux_cpuonly.yml @@ -44,6 +44,7 @@ dependencies: - plotly=4.14.3 - tabulate - simplejson + - pyparsing==3.0.9 # testing - parameterized - pip: From c04387bf29dfbd223cddca7dd4dccabedd46b23a Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 15 Oct 2024 07:51:32 -0700 Subject: [PATCH 03/11] Enable Colmap SIFT --- gtsfm/configs/correspondence/sift.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtsfm/configs/correspondence/sift.yaml b/gtsfm/configs/correspondence/sift.yaml index 6945e1352..237f07c6c 100644 --- a/gtsfm/configs/correspondence/sift.yaml +++ b/gtsfm/configs/correspondence/sift.yaml @@ -6,8 +6,8 @@ CorrespondenceGenerator: detector_descriptor: _target_: gtsfm.frontend.cacher.detector_descriptor_cacher.DetectorDescriptorCacher detector_descriptor_obj: - #_target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor - _target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor + _target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor + #_target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor max_keypoints: 5000 matcher: From 1ce2e2e029de6a04b518bd4848220bbb427cb462 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 15 Oct 2024 10:12:33 -0700 Subject: [PATCH 04/11] Use Colmap verifier --- gtsfm/configs/correspondence/sift.yaml | 4 ++-- gtsfm/configs/unified.yaml | 2 +- gtsfm/frontend/verifier/loransac.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gtsfm/configs/correspondence/sift.yaml b/gtsfm/configs/correspondence/sift.yaml index 237f07c6c..6945e1352 100644 --- a/gtsfm/configs/correspondence/sift.yaml +++ b/gtsfm/configs/correspondence/sift.yaml @@ -6,8 +6,8 @@ CorrespondenceGenerator: detector_descriptor: _target_: gtsfm.frontend.cacher.detector_descriptor_cacher.DetectorDescriptorCacher detector_descriptor_obj: - _target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor - #_target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor + #_target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor + _target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor max_keypoints: 5000 matcher: diff --git a/gtsfm/configs/unified.yaml b/gtsfm/configs/unified.yaml index 493af70cd..6ad7c8392 100644 --- a/gtsfm/configs/unified.yaml +++ b/gtsfm/configs/unified.yaml @@ -44,7 +44,7 @@ SceneOptimizer: bundle_adjust_2view_maxiters: 100 verifier: - _target_: gtsfm.frontend.verifier.ransac.Ransac + _target_: gtsfm.frontend.verifier.loransac.LoRansac use_intrinsics_in_verification: True estimation_threshold_px: 4 # for H/E/F estimators diff --git a/gtsfm/frontend/verifier/loransac.py b/gtsfm/frontend/verifier/loransac.py index b9a7f169c..2308d1154 100644 --- a/gtsfm/frontend/verifier/loransac.py +++ b/gtsfm/frontend/verifier/loransac.py @@ -27,9 +27,9 @@ MIN_INLIER_RATIO = 0.01 -MIN_NUM_TRIALS = 100000 -MAX_NUM_TRIALS = 1000000 -CONFIDENCE = 0.999999 +MIN_NUM_TRIALS = 1000 +MAX_NUM_TRIALS = 10000 +CONFIDENCE = 0.9999 class LoRansac(VerifierBase): From 1fbf2153b8fde5105634f7b609fbe1b15aabb093 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 15 Oct 2024 10:45:25 -0700 Subject: [PATCH 05/11] Increase RANSAC confidence and trials --- gtsfm/frontend/verifier/loransac.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gtsfm/frontend/verifier/loransac.py b/gtsfm/frontend/verifier/loransac.py index 2308d1154..1db8e80a9 100644 --- a/gtsfm/frontend/verifier/loransac.py +++ b/gtsfm/frontend/verifier/loransac.py @@ -27,9 +27,9 @@ MIN_INLIER_RATIO = 0.01 -MIN_NUM_TRIALS = 1000 -MAX_NUM_TRIALS = 10000 -CONFIDENCE = 0.9999 +MIN_NUM_TRIALS = 10000 +MAX_NUM_TRIALS = 100000 +CONFIDENCE = 0.999999 class LoRansac(VerifierBase): From 2aa391d8dfa10985dc8c18229c662c15193b59bf Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Wed, 16 Oct 2024 09:33:02 -0700 Subject: [PATCH 06/11] Add Colmap front-end config --- gtsfm/common/image.py | 2 +- gtsfm/common/sensor_width_database.py | 2 +- gtsfm/configs/colmap_front_end.yaml | 103 ++++++++++++++++++ gtsfm/configs/correspondence/sift.yaml | 1 - gtsfm/configs/unified.yaml | 2 +- .../detector_descriptor/colmap_sift.py | 12 +- gtsfm/frontend/verifier/loransac.py | 54 +++------ 7 files changed, 128 insertions(+), 48 deletions(-) create mode 100644 gtsfm/configs/colmap_front_end.yaml diff --git a/gtsfm/common/image.py b/gtsfm/common/image.py index 4a5d1d6c9..f7248d112 100644 --- a/gtsfm/common/image.py +++ b/gtsfm/common/image.py @@ -21,7 +21,7 @@ class Image(NamedTuple): value_array: np.ndarray exif_data: Optional[Dict[str, Any]] = None - sensor_width_db: SensorWidthDatabase = None #SensorWidthDatabase() + sensor_width_db: SensorWidthDatabase = SensorWidthDatabase() file_name: Optional[str] = None mask: Optional[np.ndarray] = None diff --git a/gtsfm/common/sensor_width_database.py b/gtsfm/common/sensor_width_database.py index f6245e1ad..04cfa37fd 100644 --- a/gtsfm/common/sensor_width_database.py +++ b/gtsfm/common/sensor_width_database.py @@ -6,7 +6,7 @@ """ from pathlib import Path -#import pandas as pd +import pandas as pd ASSETS_ROOT = Path(__file__).resolve().parent.parent / "assets" DEFAULT_SENSOR_DB_PATH = ASSETS_ROOT / "camera_details" / "sensor_database.csv" diff --git a/gtsfm/configs/colmap_front_end.yaml b/gtsfm/configs/colmap_front_end.yaml new file mode 100644 index 000000000..c36e1b075 --- /dev/null +++ b/gtsfm/configs/colmap_front_end.yaml @@ -0,0 +1,103 @@ +# Front-end configuration that leverages PyColmap for feature extraction and verfiication. + +SceneOptimizer: + _target_: gtsfm.scene_optimizer.SceneOptimizer + save_gtsfm_data: True + save_two_view_correspondences_viz: False + save_3d_viz: False + pose_angular_error_thresh: 5 # degrees + + image_pairs_generator: + _target_: gtsfm.retriever.image_pairs_generator.ImagePairsGenerator + global_descriptor: + _target_: gtsfm.frontend.cacher.global_descriptor_cacher.GlobalDescriptorCacher + global_descriptor_obj: + _target_: gtsfm.frontend.global_descriptor.netvlad_global_descriptor.NetVLADGlobalDescriptor + retriever: + _target_: gtsfm.retriever.joint_netvlad_sequential_retriever.JointNetVLADSequentialRetriever + num_matched: 5 + min_score: 0.3 + max_frame_lookahead: 15 + + correspondence_generator: + _target_: gtsfm.frontend.correspondence_generator.det_desc_correspondence_generator.DetDescCorrespondenceGenerator + + detector_descriptor: + _target_: gtsfm.frontend.cacher.detector_descriptor_cacher.DetectorDescriptorCacher + detector_descriptor_obj: + _target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor + max_keypoints: 8192 + + matcher: + _target_: gtsfm.frontend.cacher.matcher_cacher.MatcherCacher + matcher_obj: + _target_: gtsfm.frontend.matcher.twoway_matcher.TwoWayMatcher + ratio_test_threshold: 0.8 + + two_view_estimator: + _target_: gtsfm.two_view_estimator_cacher.TwoViewEstimatorCacher + two_view_estimator_obj: + _target_: gtsfm.two_view_estimator.TwoViewEstimator + bundle_adjust_2view: True + eval_threshold_px: 4 # in px + ba_reproj_error_thresholds: [0.5] + bundle_adjust_2view_maxiters: 100 + + verifier: + _target_: gtsfm.frontend.verifier.loransac.LoRansac + use_intrinsics_in_verification: True + estimation_threshold_px: 4 # for H/E/F estimators + + triangulation_options: + _target_: gtsfm.data_association.point3d_initializer.TriangulationOptions + mode: + _target_: gtsfm.data_association.point3d_initializer.TriangulationSamplingMode + value: NO_RANSAC + + inlier_support_processor: + _target_: gtsfm.two_view_estimator.InlierSupportProcessor + min_num_inliers_est_model: 15 + min_inlier_ratio_est_model: 0.1 + + multiview_optimizer: + _target_: gtsfm.multi_view_optimizer.MultiViewOptimizer + + # comment out to not run + view_graph_estimator: + _target_: gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator.CycleConsistentRotationViewGraphEstimator + edge_error_aggregation_criterion: MIN_EDGE_ERROR + + rot_avg_module: + _target_: gtsfm.averaging.rotation.shonan.ShonanRotationAveraging + weight_by_inliers: True + + trans_avg_module: + _target_: gtsfm.averaging.translation.averaging_1dsfm.TranslationAveraging1DSFM + robust_measurement_noise: True + projection_sampling_method: SAMPLE_INPUT_MEASUREMENTS + reject_outliers: True + + data_association_module: + _target_: gtsfm.data_association.data_assoc.DataAssociation + min_track_len: 3 + triangulation_options: + _target_: gtsfm.data_association.point3d_initializer.TriangulationOptions + reproj_error_threshold: 10 + mode: + _target_: gtsfm.data_association.point3d_initializer.TriangulationSamplingMode + value: RANSAC_SAMPLE_UNIFORM + max_num_hypotheses: 100 + save_track_patches_viz: False + + bundle_adjustment_module: + _target_: gtsfm.bundle.bundle_adjustment.BundleAdjustmentOptimizer + reproj_error_thresholds: [10, 5, 3] # for (multistage) post-optimization filtering + robust_measurement_noise: True + shared_calib: False + cam_pose3_prior_noise_sigma: 0.1 + calibration_prior_noise_sigma: 1e-5 + measurement_noise_sigma: 1.0 + + # comment out to not run + dense_multiview_optimizer: + _target_: gtsfm.densify.mvs_patchmatchnet.MVSPatchmatchNet diff --git a/gtsfm/configs/correspondence/sift.yaml b/gtsfm/configs/correspondence/sift.yaml index 6945e1352..8691035a6 100644 --- a/gtsfm/configs/correspondence/sift.yaml +++ b/gtsfm/configs/correspondence/sift.yaml @@ -6,7 +6,6 @@ CorrespondenceGenerator: detector_descriptor: _target_: gtsfm.frontend.cacher.detector_descriptor_cacher.DetectorDescriptorCacher detector_descriptor_obj: - #_target_: gtsfm.frontend.detector_descriptor.colmap_sift.ColmapSIFTDetectorDescriptor _target_: gtsfm.frontend.detector_descriptor.sift.SIFTDetectorDescriptor max_keypoints: 5000 diff --git a/gtsfm/configs/unified.yaml b/gtsfm/configs/unified.yaml index 6ad7c8392..493af70cd 100644 --- a/gtsfm/configs/unified.yaml +++ b/gtsfm/configs/unified.yaml @@ -44,7 +44,7 @@ SceneOptimizer: bundle_adjust_2view_maxiters: 100 verifier: - _target_: gtsfm.frontend.verifier.loransac.LoRansac + _target_: gtsfm.frontend.verifier.ransac.Ransac use_intrinsics_in_verification: True estimation_threshold_px: 4 # for H/E/F estimators diff --git a/gtsfm/frontend/detector_descriptor/colmap_sift.py b/gtsfm/frontend/detector_descriptor/colmap_sift.py index 2f8795f84..18ab53370 100644 --- a/gtsfm/frontend/detector_descriptor/colmap_sift.py +++ b/gtsfm/frontend/detector_descriptor/colmap_sift.py @@ -44,15 +44,11 @@ def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: # TODO (travisdriver): Add GPU support colmap_obj = pycolmap.Sift() - # Run the OpenCV code. - features, descriptors = colmap_obj.extract(gray_image.value_array) + # Extract features. + features, descriptors = colmap_obj.extract(gray_image.value_array, max_num_features=self.max_keypoints) # Convert to GTSFM's keypoints. - # TODO (travisdriver): Add scales and orientations - keypoints = Keypoints(coordinates=features[..., :2]) - - # # Filter features. - # keypoints, selection_idxs = keypoints.get_top_k(self.max_keypoints) - # descriptors = descriptors[selection_idxs] + # Note: Columns of features is x-coordinate, y-coordinate, scale, and orientation, respectively. + keypoints = Keypoints(coordinates=features[..., :2], scales=features[:, 2]) return keypoints, descriptors diff --git a/gtsfm/frontend/verifier/loransac.py b/gtsfm/frontend/verifier/loransac.py index 1db8e80a9..11606ceca 100644 --- a/gtsfm/frontend/verifier/loransac.py +++ b/gtsfm/frontend/verifier/loransac.py @@ -26,10 +26,11 @@ logger = logger_utils.get_logger() +# Default Colmap params. MIN_INLIER_RATIO = 0.01 -MIN_NUM_TRIALS = 10000 -MAX_NUM_TRIALS = 100000 -CONFIDENCE = 0.999999 +MIN_NUM_TRIALS = 1000 +MAX_NUM_TRIALS = 10000 +CONFIDENCE = 0.9999 class LoRansac(VerifierBase): @@ -37,6 +38,10 @@ def __init__( self, use_intrinsics_in_verification: bool, estimation_threshold_px: float, + min_inlier_ratio: float = MIN_INLIER_RATIO, + min_num_trials: int = MIN_NUM_TRIALS, + max_num_trials: int = MAX_NUM_TRIALS, + confidence: float = CONFIDENCE, ) -> None: """Initializes the verifier. @@ -50,15 +55,16 @@ def __init__( Sampson distance. """ super().__init__(use_intrinsics_in_verification, estimation_threshold_px) - self._min_matches = ( - verifier_base.NUM_MATCHES_REQ_E_MATRIX - if self._use_intrinsics_in_verification - else verifier_base.NUM_MATCHES_REQ_F_MATRIX + self._ransac_options = pycolmap.RANSACOptions( + { + "max_error": self._estimation_threshold_px, + "min_num_trials": min_num_trials, + "max_num_trials": max_num_trials, + "min_inlier_ratio": min_inlier_ratio, + "confidence": confidence, + } ) - # for failure, i2Ri1 = None, and i2Ui1 = None, and no verified correspondences, and inlier_ratio_est_model = 0 - self._failure_result = (None, None, np.array([], dtype=np.uint64), 0.0) - def __estimate_essential_matrix( self, uv_i1: np.ndarray, @@ -81,19 +87,7 @@ def __estimate_essential_matrix( camera_i2: pycolmap.Camera = pycolmap_utils.get_pycolmap_camera(camera_intrinsics_i2) result_dict = pycolmap.essential_matrix_estimation( - uv_i1, - uv_i2, - camera_i1, - camera_i2, - pycolmap.RANSACOptions( - { - "max_error": self._estimation_threshold_px, - "min_num_trials": MIN_NUM_TRIALS, - "max_num_trials": MAX_NUM_TRIALS, - "min_inlier_ratio": MIN_INLIER_RATIO, - "confidence": CONFIDENCE, - } - ), + uv_i1, uv_i2, camera_i1, camera_i2, self._ransac_options, ) return result_dict @@ -131,19 +125,7 @@ def verify( if self._use_intrinsics_in_verification: result_dict = self.__estimate_essential_matrix(uv_i1, uv_i2, camera_intrinsics_i1, camera_intrinsics_i2) else: - result_dict = pycolmap.fundamental_matrix_estimation( - uv_i1, - uv_i2, - pycolmap.RANSACOptions( - { - "max_error": self._estimation_threshold_px, - "min_num_trials": MIN_NUM_TRIALS, - "max_num_trials": MAX_NUM_TRIALS, - "min_inlier_ratio": MIN_INLIER_RATIO, - "confidence": CONFIDENCE, - } - ), - ) + result_dict = pycolmap.fundamental_matrix_estimation(uv_i1, uv_i2, self._ransac_options) if not result_dict: matrix_type = "Essential" if self._use_intrinsics_in_verification else "Fundamental" From 853e9ad247ee8b1ac8b5716017ff33600c7c05a1 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Wed, 16 Oct 2024 12:21:00 -0700 Subject: [PATCH 07/11] Add SiftExtractionOptions; switch to exhaustive matching --- gtsfm/configs/colmap_front_end.yaml | 13 +------------ gtsfm/frontend/detector_descriptor/colmap_sift.py | 9 +++++---- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/gtsfm/configs/colmap_front_end.yaml b/gtsfm/configs/colmap_front_end.yaml index c36e1b075..71ba679dd 100644 --- a/gtsfm/configs/colmap_front_end.yaml +++ b/gtsfm/configs/colmap_front_end.yaml @@ -9,15 +9,8 @@ SceneOptimizer: image_pairs_generator: _target_: gtsfm.retriever.image_pairs_generator.ImagePairsGenerator - global_descriptor: - _target_: gtsfm.frontend.cacher.global_descriptor_cacher.GlobalDescriptorCacher - global_descriptor_obj: - _target_: gtsfm.frontend.global_descriptor.netvlad_global_descriptor.NetVLADGlobalDescriptor retriever: - _target_: gtsfm.retriever.joint_netvlad_sequential_retriever.JointNetVLADSequentialRetriever - num_matched: 5 - min_score: 0.3 - max_frame_lookahead: 15 + _target_: gtsfm.retriever.exhaustive_retriever.ExhaustiveRetriever correspondence_generator: _target_: gtsfm.frontend.correspondence_generator.det_desc_correspondence_generator.DetDescCorrespondenceGenerator @@ -97,7 +90,3 @@ SceneOptimizer: cam_pose3_prior_noise_sigma: 0.1 calibration_prior_noise_sigma: 1e-5 measurement_noise_sigma: 1.0 - - # comment out to not run - dense_multiview_optimizer: - _target_: gtsfm.densify.mvs_patchmatchnet.MVSPatchmatchNet diff --git a/gtsfm/frontend/detector_descriptor/colmap_sift.py b/gtsfm/frontend/detector_descriptor/colmap_sift.py index 18ab53370..2587b4d77 100644 --- a/gtsfm/frontend/detector_descriptor/colmap_sift.py +++ b/gtsfm/frontend/detector_descriptor/colmap_sift.py @@ -22,7 +22,7 @@ class ColmapSIFTDetectorDescriptor(DetectorDescriptorBase): - """SIFT detector-descriptor using OpenCV's implementation.""" + """SIFT detector-descriptor using Colmap's implementation.""" def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: """Perform feature detection as well as their description. @@ -40,12 +40,13 @@ def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: # Convert to grayscale. gray_image = image_utils.rgb_to_gray_cv(image) - # Create OpenCV object every time as the object is not pickle-able. + # Create pycolmap object every time as the object is not pickle-able. # TODO (travisdriver): Add GPU support - colmap_obj = pycolmap.Sift() + options = pycolmap.SiftExtractionOptions(max_num_features=self.max_keypoints) + colmap_obj = pycolmap.Sift(options) # Extract features. - features, descriptors = colmap_obj.extract(gray_image.value_array, max_num_features=self.max_keypoints) + features, descriptors = colmap_obj.extract(gray_image.value_array.astype(np.float32)) # Convert to GTSFM's keypoints. # Note: Columns of features is x-coordinate, y-coordinate, scale, and orientation, respectively. From 90a8a5fa69d01ceb142d6ec17f2028fec268336d Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 5 Nov 2024 12:35:07 -0800 Subject: [PATCH 08/11] Fix unused imports --- gtsfm/frontend/detector_descriptor/colmap_sift.py | 3 +-- gtsfm/frontend/verifier/loransac.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gtsfm/frontend/detector_descriptor/colmap_sift.py b/gtsfm/frontend/detector_descriptor/colmap_sift.py index 2587b4d77..a60b1045b 100644 --- a/gtsfm/frontend/detector_descriptor/colmap_sift.py +++ b/gtsfm/frontend/detector_descriptor/colmap_sift.py @@ -14,7 +14,6 @@ import numpy as np import pycolmap -import gtsfm.utils.features as feature_utils import gtsfm.utils.images as image_utils from gtsfm.common.image import Image from gtsfm.common.keypoints import Keypoints @@ -41,7 +40,7 @@ def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: gray_image = image_utils.rgb_to_gray_cv(image) # Create pycolmap object every time as the object is not pickle-able. - # TODO (travisdriver): Add GPU support + # Note: cannot use SiftGPU as wheels are not built with CUDA support. options = pycolmap.SiftExtractionOptions(max_num_features=self.max_keypoints) colmap_obj = pycolmap.Sift(options) diff --git a/gtsfm/frontend/verifier/loransac.py b/gtsfm/frontend/verifier/loransac.py index 11606ceca..23d28961a 100644 --- a/gtsfm/frontend/verifier/loransac.py +++ b/gtsfm/frontend/verifier/loransac.py @@ -16,7 +16,6 @@ import pycolmap from gtsam import Cal3Bundler, Rot3, Unit3 -import gtsfm.frontend.verifier.verifier_base as verifier_base import gtsfm.utils.logger as logger_utils import gtsfm.utils.pycolmap_utils as pycolmap_utils import gtsfm.utils.verification as verification_utils From 4f7b65876f4058ee3b41b37f78a1318b93c7cb6d Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 5 Nov 2024 12:36:27 -0800 Subject: [PATCH 09/11] Change variable name --- gtsfm/frontend/detector_descriptor/colmap_sift.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtsfm/frontend/detector_descriptor/colmap_sift.py b/gtsfm/frontend/detector_descriptor/colmap_sift.py index a60b1045b..de9de8ee8 100644 --- a/gtsfm/frontend/detector_descriptor/colmap_sift.py +++ b/gtsfm/frontend/detector_descriptor/colmap_sift.py @@ -42,10 +42,10 @@ def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]: # Create pycolmap object every time as the object is not pickle-able. # Note: cannot use SiftGPU as wheels are not built with CUDA support. options = pycolmap.SiftExtractionOptions(max_num_features=self.max_keypoints) - colmap_obj = pycolmap.Sift(options) + sift_obj = pycolmap.Sift(options) # Extract features. - features, descriptors = colmap_obj.extract(gray_image.value_array.astype(np.float32)) + features, descriptors = sift_obj.extract(gray_image.value_array.astype(np.float32)) # Convert to GTSFM's keypoints. # Note: Columns of features is x-coordinate, y-coordinate, scale, and orientation, respectively. From 217b05d0296c5e0c258641901d90ea427395e2e4 Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 5 Nov 2024 12:46:16 -0800 Subject: [PATCH 10/11] Fix Kornia version --- environment_linux.yml | 2 +- environment_linux_cpuonly.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_linux.yml b/environment_linux.yml index 18cb81980..6ae134c41 100644 --- a/environment_linux.yml +++ b/environment_linux.yml @@ -38,7 +38,7 @@ dependencies: - pytorch-cuda=11.8 - pytorch - torchvision>=0.13.0 - - kornia + - kornia==0.7.3 - pycolmap # io - h5py diff --git a/environment_linux_cpuonly.yml b/environment_linux_cpuonly.yml index 6a21b640e..953c0e0da 100644 --- a/environment_linux_cpuonly.yml +++ b/environment_linux_cpuonly.yml @@ -37,7 +37,7 @@ dependencies: - cpuonly # replacement of cudatoolkit for cpu only machines - pytorch>=1.12.0 - torchvision>=0.13.0 - - kornia + - kornia==0.7.3 - pycolmap # io - h5py From 02bf2d0dec39d13301b37025a2918206af19b42e Mon Sep 17 00:00:00 2001 From: Travis Driver Date: Tue, 5 Nov 2024 14:43:43 -0800 Subject: [PATCH 11/11] Fix ColmapSift unit test and update documentation --- .../detector_descriptor/test_colmap_sift.py | 6 +++- .../test_colmap_sift_detector_descriptor.py | 31 ------------------- 2 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 tests/frontend/detector_descriptor/test_colmap_sift_detector_descriptor.py diff --git a/tests/frontend/detector_descriptor/test_colmap_sift.py b/tests/frontend/detector_descriptor/test_colmap_sift.py index 55dbb4296..8e66fea46 100644 --- a/tests/frontend/detector_descriptor/test_colmap_sift.py +++ b/tests/frontend/detector_descriptor/test_colmap_sift.py @@ -18,7 +18,11 @@ class TestColmapSIFTDetectorDescriptor(test_detector_descriptor_base.TestDetecto def setUp(self): """Setup the attributes for the tests.""" super().setUp() - self.detector_descriptor = ColmapSIFTDetectorDescriptor() + # Note: pycolmap does not guarantee that the number of keypoints will not exceed the specified maximum, as there + # can be ties in terms of the response scores. E.g., if the 5000th keypoint and the 5001st keypoint have the + # same response, pycolmap will return 5001 keypoints. Setting the number of maximum keypoints lower reduces the + # risk of this happening. + self.detector_descriptor = ColmapSIFTDetectorDescriptor(max_keypoints=2000) # explicitly set the detector self.detector = DetectorFromDetectorDescriptor(self.detector_descriptor) diff --git a/tests/frontend/detector_descriptor/test_colmap_sift_detector_descriptor.py b/tests/frontend/detector_descriptor/test_colmap_sift_detector_descriptor.py deleted file mode 100644 index 09d4650a3..000000000 --- a/tests/frontend/detector_descriptor/test_colmap_sift_detector_descriptor.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Tests for SIFT detector descriptor - -Authors: Ayush Baid -""" -# import unittest - -# import tests.frontend.detector_descriptor.test_detector_descriptor_base as test_detector_descriptor_base -# from gtsfm.frontend.detector.detector_from_joint_detector_descriptor import ( -# DetectorFromDetectorDescriptor, -# ) - -# from gtsfm.frontend.detector_descriptor.colmap_sift import ColmapSIFTDetectorDescriptor - - -# class TestColmapSIFTDetectorDescriptor(test_detector_descriptor_base.TestDetectorDescriptorBase): -# """Unit tests for ColmapSIFT. - -# All unit test functions defined in TestDetectorDescriptorBase are run automatically. -# """ - -# def setUp(self): -# """Setup the attributes for the tests.""" -# super().setUp() -# self.detector_descriptor = ColmapSIFTDetectorDescriptor() - -# # explicitly set the detector -# self.detector = DetectorFromDetectorDescriptor(self.detector_descriptor) - - -# if __name__ == "__main__": -# unittest.main()