From 2d082a99b81b4c8d59aa16302021416724b6ae7a Mon Sep 17 00:00:00 2001 From: Akshay Krishnan Date: Fri, 24 May 2024 15:30:31 +0000 Subject: [PATCH] Revert "Revert "Merge branch 'master' of github.com:borglab/gtsfm"" This reverts commit 6abf345522a215e25d34c0198bee0ae87eb856fa. --- .../rotation/rotation_averaging_base.py | 37 +- gtsfm/averaging/rotation/shonan.py | 100 +++-- gtsfm/configs/unified.yaml | 4 +- gtsfm/frontend/verifier/verifier_base.py | 6 + gtsfm/multi_view_optimizer.py | 35 +- gtsfm/two_view_estimator_cacher.py | 4 +- rtf_vis_tool/package-lock.json | 120 ++--- scripts/benchmark_wildcat.sh | 422 +++++++++--------- scripts/eth3d_benchmark.sh | 182 ++++---- tests/averaging/rotation/test_shonan.py | 31 +- tests/test_two_view_estimator_cacher.py | 16 +- 11 files changed, 544 insertions(+), 413 deletions(-) diff --git a/gtsfm/averaging/rotation/rotation_averaging_base.py b/gtsfm/averaging/rotation/rotation_averaging_base.py index feb1b3597..a98912557 100644 --- a/gtsfm/averaging/rotation/rotation_averaging_base.py +++ b/gtsfm/averaging/rotation/rotation_averaging_base.py @@ -8,6 +8,7 @@ from typing import Dict, List, Optional, Tuple import dask +import numpy as np from dask.delayed import Delayed from gtsam import Pose3, Rot3 @@ -42,13 +43,15 @@ def run_rotation_averaging( num_images: int, i2Ri1_dict: Dict[Tuple[int, int], Optional[Rot3]], i1Ti2_priors: Dict[Tuple[int, int], PosePrior], + v_corr_idxs: Dict[Tuple[int, int], np.ndarray], ) -> List[Optional[Rot3]]: """Run the rotation averaging. Args: - num_images: number of poses. - i2Ri1_dict: relative rotations as dictionary (i1, i2): i2Ri1. - i1Ti2_priors: priors on relative poses as dictionary(i1, i2): PosePrior on i1Ti2. + num_images: Number of poses. + i2Ri1_dict: Relative rotations as dictionary (i1, i2): i2Ri1. + i1Ti2_priors: Priors on relative poses as dictionary(i1, i2): PosePrior on i1Ti2. + v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. Returns: Global rotations for each camera pose, i.e. wRi, as a list. The number of entries in the list is @@ -61,6 +64,7 @@ def _run_rotation_averaging_base( num_images: int, i2Ri1_dict: Dict[Tuple[int, int], Optional[Rot3]], i1Ti2_priors: Dict[Tuple[int, int], PosePrior], + v_corr_idxs: Dict[Tuple[int, int], np.ndarray], wTi_gt: List[Optional[Pose3]], ) -> Tuple[List[Optional[Rot3]], GtsfmMetricsGroup]: """Runs rotation averaging and computes metrics. @@ -69,6 +73,7 @@ def _run_rotation_averaging_base( num_images: Number of poses. i2Ri1_dict: Relative rotations as dictionary (i1, i2): i2Ri1. i1Ti2_priors: Priors on relative poses as dictionary(i1, i2): PosePrior on i1Ti2. + v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. wTi_gt: Ground truth global rotations to compare against. Returns: @@ -78,7 +83,7 @@ def _run_rotation_averaging_base( Metrics on global rotations. """ start_time = time.time() - wRis = self.run_rotation_averaging(num_images, i2Ri1_dict, i1Ti2_priors) + wRis = self.run_rotation_averaging(num_images, i2Ri1_dict, i1Ti2_priors, v_corr_idxs) run_time = time.time() - start_time metrics = self.evaluate(wRis, wTi_gt) @@ -93,11 +98,11 @@ def evaluate(self, wRi_computed: List[Optional[Rot3]], wTi_gt: List[Optional[Pos wRi_computed: List of global rotations computed. wTi_gt: Ground truth global rotations to compare against. - Raises: - ValueError: If the length of the computed and GT list differ. - Returns: Metrics on global rotations. + + Raises: + ValueError: If the length of the computed and GT list differ. """ wRi_gt = [wTi.rotation() if wTi is not None else None for wTi in wTi_gt] @@ -116,22 +121,28 @@ def create_computation_graph( num_images: int, i2Ri1_graph: Delayed, i1Ti2_priors: Dict[Tuple[int, int], PosePrior], + v_corr_idxs: Dict[Tuple[int, int], np.ndarray], gt_wTi_list: List[Optional[Pose3]], ) -> Tuple[Delayed, Delayed]: """Create the computation graph for performing rotation averaging. Args: - num_images: number of poses. - i2Ri1_graph: dictionary of relative rotations as a delayed task. - i1Ti2_priors: priors on relative poses as (i1, i2): PosePrior on i1Ti2. - gt_wTi_list: ground truth poses, to be used for evaluation. + num_images: Number of poses. + i2Ri1_graph: Dictionary of relative rotations as a delayed task. + i1Ti2_priors: Priors on relative poses as (i1, i2): PosePrior on i1Ti2. + v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. + gt_wTi_list: Ground truth poses, to be used for evaluation. Returns: - global rotations wrapped using dask.delayed. + Global rotations wrapped using dask.delayed. """ wRis, metrics = dask.delayed(self._run_rotation_averaging_base, nout=2)( - num_images, i2Ri1_dict=i2Ri1_graph, i1Ti2_priors=i1Ti2_priors, wTi_gt=gt_wTi_list + num_images, + i2Ri1_dict=i2Ri1_graph, + i1Ti2_priors=i1Ti2_priors, + v_corr_idxs=v_corr_idxs, + wTi_gt=gt_wTi_list, ) return wRis, metrics diff --git a/gtsfm/averaging/rotation/shonan.py b/gtsfm/averaging/rotation/shonan.py index 08f2c3dc6..ad8643a43 100644 --- a/gtsfm/averaging/rotation/shonan.py +++ b/gtsfm/averaging/rotation/shonan.py @@ -16,10 +16,7 @@ import gtsam import numpy as np from gtsam import ( - BetweenFactorPose3, - BetweenFactorPose3s, LevenbergMarquardtParams, - Pose3, Rot3, ShonanAveraging3, ShonanAveragingParameters3, @@ -29,6 +26,7 @@ from gtsfm.averaging.rotation.rotation_averaging_base import RotationAveragingBase from gtsfm.common.pose_prior import PosePrior +ROT3_DOF = 3 POSE3_DOF = 6 logger = logger_utils.get_logger() @@ -39,18 +37,23 @@ class ShonanRotationAveraging(RotationAveragingBase): """Performs Shonan rotation averaging.""" - def __init__(self, two_view_rotation_sigma: float = _DEFAULT_TWO_VIEW_ROTATION_SIGMA) -> None: + def __init__( + self, two_view_rotation_sigma: float = _DEFAULT_TWO_VIEW_ROTATION_SIGMA, weight_by_inliers: bool = True + ) -> None: """Initializes module. Note: `p_min` and `p_max` describe the minimum and maximum relaxation rank. Args: two_view_rotation_sigma: Covariance to use (lower values -> more strictly adhere to input measurements). + weight_by_inliers: Whether to weight pairwise costs according to an uncertainty equal to the inverse number + of inlier correspondences per edge. """ super().__init__() self._two_view_rotation_sigma = two_view_rotation_sigma self._p_min = 3 self._p_max = 64 + self._weight_by_inliers = weight_by_inliers def __get_shonan_params(self) -> ShonanAveragingParameters3: lm_params = LevenbergMarquardtParams.CeresDefaults() @@ -59,30 +62,34 @@ def __get_shonan_params(self) -> ShonanAveragingParameters3: shonan_params.setCertifyOptimality(True) return shonan_params - def __between_factors_from_2view_relative_rotations( - self, i2Ri1_dict: Dict[Tuple[int, int], Rot3], old_to_new_idxs: Dict[int, int] - ) -> BetweenFactorPose3s: + def __measurements_from_2view_relative_rotations( + self, + i2Ri1_dict: Dict[Tuple[int, int], Rot3], + num_correspondences_dict: Dict[Tuple[int, int], int], + ) -> gtsam.BinaryMeasurementsRot3: """Create between factors from relative rotations computed by the 2-view estimator.""" # TODO: how to weight the noise model on relative rotations compared to priors? - noise_model = gtsam.noiseModel.Isotropic.Sigma(POSE3_DOF, self._two_view_rotation_sigma) - between_factors = BetweenFactorPose3s() + # Default noise model if `self._weight_by_inliers` is False, or zero correspondences on edge. + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, self._two_view_rotation_sigma) + measurements = gtsam.BinaryMeasurementsRot3() for (i1, i2), i2Ri1 in i2Ri1_dict.items(): - if i2Ri1 is not None: + if i2Ri1 is None: + continue + if self._weight_by_inliers and num_correspondences_dict[(i1, i2)] > 0: # ignore translation during rotation averaging - i2Ti1 = Pose3(i2Ri1, np.zeros(3)) - i2_ = old_to_new_idxs[i2] - i1_ = old_to_new_idxs[i1] - between_factors.append(BetweenFactorPose3(i2_, i1_, i2Ti1, noise_model)) + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, 1 / num_correspondences_dict[(i1, i2)]) + + measurements.append(gtsam.BinaryMeasurementRot3(i2, i1, i2Ri1, noise_model)) - return between_factors + return measurements - def _between_factors_from_pose_priors( + def _measurements_from_pose_priors( self, i1Ti2_priors: Dict[Tuple[int, int], PosePrior], old_to_new_idxs: Dict[int, int] - ) -> BetweenFactorPose3s: + ) -> gtsam.BinaryMeasurementsRot3: """Create between factors from the priors on relative poses.""" - between_factors = BetweenFactorPose3s() + measurements = gtsam.BinaryMeasurementsRot3() def get_isotropic_noise_model_sigma(covariance: np.ndarray) -> float: """Get the sigma to be used for the isotropic noise model. @@ -95,13 +102,13 @@ def get_isotropic_noise_model_sigma(covariance: np.ndarray) -> float: i1_ = old_to_new_idxs[i1] i2_ = old_to_new_idxs[i2] noise_model_sigma = get_isotropic_noise_model_sigma(i1Ti2_prior.covariance) - noise_model = gtsam.noiseModel.Isotropic.Sigma(POSE3_DOF, noise_model_sigma) - between_factors.append(BetweenFactorPose3(i1_, i2_, i1Ti2_prior.value, noise_model)) + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, noise_model_sigma) + measurements.append(gtsam.BinaryMeasurementRot3(i1_, i2_, i1Ti2_prior.value.rotation(), noise_model)) - return between_factors + return measurements def _run_with_consecutive_ordering( - self, num_connected_nodes: int, between_factors: BetweenFactorPose3s + self, num_connected_nodes: int, measurements: gtsam.BinaryMeasurementsRot3 ) -> List[Optional[Rot3]]: """Run the rotation averaging on a connected graph w/ N keys ordered consecutively [0,...,N-1]. @@ -112,7 +119,7 @@ def _run_with_consecutive_ordering( Args: num_connected_nodes: Number of unique connected nodes (i.e. images) in the graph (<= the number of images in the dataset) - between_factors: BetweenFactorPose3s created from relative rotations from 2-view estimator and the priors. + measurements: BinaryMeasurementsRot3 created from relative rotations from 2-view estimator and the priors. Returns: Global rotations for each **CONNECTED** camera pose, i.e. wRi, as a list. The number of entries in @@ -122,10 +129,10 @@ def _run_with_consecutive_ordering( logger.info( "Running Shonan with %d constraints on %d nodes", - len(between_factors), + len(measurements), num_connected_nodes, ) - shonan = ShonanAveraging3(between_factors, self.__get_shonan_params()) + shonan = ShonanAveraging3(measurements, self.__get_shonan_params()) initial = shonan.initializeRandomly() logger.info("Initial cost: %.5f", shonan.cost(initial)) @@ -159,6 +166,7 @@ def run_rotation_averaging( num_images: int, i2Ri1_dict: Dict[Tuple[int, int], Optional[Rot3]], i1Ti2_priors: Dict[Tuple[int, int], PosePrior], + v_corr_idxs: Dict[Tuple[int, int], np.ndarray], ) -> List[Optional[Rot3]]: """Run the rotation averaging on a connected graph with arbitrary keys, where each key is a image/pose index. @@ -170,6 +178,7 @@ def run_rotation_averaging( num_images: Number of images. Since we have one pose per image, it is also the number of poses. i2Ri1_dict: Relative rotations for each image pair-edge as dictionary (i1, i2): i2Ri1. i1Ti2_priors: Priors on relative poses. + v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. Returns: Global rotations for each camera pose, i.e. wRi, as a list. The number of entries in the list is @@ -183,17 +192,36 @@ def run_rotation_averaging( return wRi_list nodes_with_edges = sorted(list(self._nodes_with_edges(i2Ri1_dict, i1Ti2_priors))) - old_to_new_idxes = {old_idx: i for i, old_idx in enumerate(nodes_with_edges)} - - between_factors: BetweenFactorPose3s = self.__between_factors_from_2view_relative_rotations( - i2Ri1_dict, old_to_new_idxes - ) - between_factors.extend(self._between_factors_from_pose_priors(i1Ti2_priors, old_to_new_idxes)) - - wRi_list_subset = self._run_with_consecutive_ordering( - num_connected_nodes=len(nodes_with_edges), between_factors=between_factors - ) - + old_to_new_idxs = {old_idx: i for i, old_idx in enumerate(nodes_with_edges)} + + i2Ri1_dict_remapped = { + (old_to_new_idxs[i1], old_to_new_idxs[i2]): i2Ri1 for (i1, i2), i2Ri1 in i2Ri1_dict.items() + } + num_correspondences_dict: Dict[Tuple[int, int], int] = { + (old_to_new_idxs[i1], old_to_new_idxs[i2]): len(v_corr_idxs[(i1, i2)]) + for (i1, i2) in v_corr_idxs.keys() + if (i1, i2) in i2Ri1_dict + } + + def _create_factors_and_run() -> List[Rot3]: + measurements: gtsam.BinaryMeasurementsRot3 = self.__measurements_from_2view_relative_rotations( + i2Ri1_dict=i2Ri1_dict_remapped, num_correspondences_dict=num_correspondences_dict + ) + measurements.extend(self._measurements_from_pose_priors(i1Ti2_priors, old_to_new_idxs)) + wRi_list_subset = self._run_with_consecutive_ordering( + num_connected_nodes=len(nodes_with_edges), measurements=measurements + ) + return wRi_list_subset + + try: + wRi_list_subset = _create_factors_and_run() + except RuntimeError: + logger.exception("Shonan failed") + if self._weight_by_inliers is True: + logger.info("Reattempting Shonan without inlier-weighted costs...") + # At times, Shonan's `SparseMinimumEigenValue` fails to compute minimum eigenvalue. + self._weight_by_inliers = False + wRi_list_subset = _create_factors_and_run() wRi_list = [None] * num_images for remapped_i, original_i in enumerate(nodes_with_edges): wRi_list[original_i] = wRi_list_subset[remapped_i] diff --git a/gtsfm/configs/unified.yaml b/gtsfm/configs/unified.yaml index 8fef52586..493af70cd 100644 --- a/gtsfm/configs/unified.yaml +++ b/gtsfm/configs/unified.yaml @@ -65,15 +65,17 @@ SceneOptimizer: # comment out to not run view_graph_estimator: _target_: gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator.CycleConsistentRotationViewGraphEstimator - edge_error_aggregation_criterion: MEDIAN_EDGE_ERROR + edge_error_aggregation_criterion: MIN_EDGE_ERROR rot_avg_module: _target_: gtsfm.averaging.rotation.shonan.ShonanRotationAveraging + weight_by_inliers: True trans_avg_module: _target_: gtsfm.averaging.translation.averaging_1dsfm.TranslationAveraging1DSFM robust_measurement_noise: True projection_sampling_method: SAMPLE_INPUT_MEASUREMENTS + reject_outliers: True data_association_module: _target_: gtsfm.data_association.data_assoc.DataAssociation diff --git a/gtsfm/frontend/verifier/verifier_base.py b/gtsfm/frontend/verifier/verifier_base.py index ef670398c..af1a45d2b 100644 --- a/gtsfm/frontend/verifier/verifier_base.py +++ b/gtsfm/frontend/verifier/verifier_base.py @@ -33,6 +33,12 @@ def get_ui_metadata() -> UiMetadata: parent_plate="Two-View Estimator", ) + def __repr__(self) -> str: + return ( + f"{type(self).__name__}" + + f"__use_intrinsics{self._use_intrinsics_in_verification}_{self._estimation_threshold_px}px" + ) + def __init__( self, use_intrinsics_in_verification: bool, diff --git a/gtsfm/multi_view_optimizer.py b/gtsfm/multi_view_optimizer.py index e6c557bac..c5d23e6d2 100644 --- a/gtsfm/multi_view_optimizer.py +++ b/gtsfm/multi_view_optimizer.py @@ -21,11 +21,15 @@ from gtsfm.common.keypoints import Keypoints from gtsfm.common.pose_prior import PosePrior from gtsfm.common.sfm_track import SfmTrack2d +from gtsfm.common.two_view_estimation_report import TwoViewEstimationReport +from gtsfm.data_association.cpp_dsf_tracks_estimator import CppDsfTracksEstimator from gtsfm.data_association.data_assoc import DataAssociation from gtsfm.evaluation.metrics import GtsfmMetricsGroup +from gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator import ( + CycleConsistentRotationViewGraphEstimator, + EdgeErrorAggregationCriterion, +) from gtsfm.view_graph_estimator.view_graph_estimator_base import ViewGraphEstimatorBase -from gtsfm.data_association.cpp_dsf_tracks_estimator import CppDsfTracksEstimator -from gtsfm.common.two_view_estimation_report import TwoViewEstimationReport class MultiViewOptimizer: @@ -44,6 +48,10 @@ def __init__( self.ba_optimizer = bundle_adjustment_module self._run_view_graph_estimator: bool = self.view_graph_estimator is not None + self.view_graph_estimator_v2 = CycleConsistentRotationViewGraphEstimator( + edge_error_aggregation_criterion=EdgeErrorAggregationCriterion.MEDIAN_EDGE_ERROR + ) + def __repr__(self) -> str: return f""" MultiviewOptimizer: @@ -63,7 +71,7 @@ def create_computation_graph( all_intrinsics: List[Optional[gtsfm_types.CALIBRATION_TYPE]], absolute_pose_priors: List[Optional[PosePrior]], relative_pose_priors: Dict[Tuple[int, int], PosePrior], - two_view_reports_dict: Optional[Dict[Tuple[int, int], TwoViewEstimationReport]], + two_view_reports_dict: Dict[Tuple[int, int], TwoViewEstimationReport], cameras_gt: List[Optional[gtsfm_types.CAMERA_TYPE]], gt_wTi_list: List[Optional[Pose3]], output_root: Optional[Path] = None, @@ -114,6 +122,21 @@ def create_computation_graph( two_view_reports_dict, debug_output_dir, ) + ( + viewgraph_i2Ri1_graph, + viewgraph_i2Ui1_graph, + viewgraph_v_corr_idxs_graph, + viewgraph_two_view_reports_graph, + viewgraph_estimation_metrics, + ) = self.view_graph_estimator_v2.create_computation_graph( + viewgraph_i2Ri1_graph, + viewgraph_i2Ui1_graph, + all_intrinsics, + viewgraph_v_corr_idxs_graph, + keypoints_list, + viewgraph_two_view_reports_graph, + debug_output_dir / "2", + ) else: viewgraph_i2Ri1_graph = dask.delayed(i2Ri1_dict) viewgraph_i2Ui1_graph = dask.delayed(i2Ui1_dict) @@ -126,7 +149,11 @@ def create_computation_graph( viewgraph_i2Ri1_graph, viewgraph_i2Ui1_graph, relative_pose_priors ) delayed_wRi, rot_avg_metrics = self.rot_avg_module.create_computation_graph( - num_images, pruned_i2Ri1_graph, i1Ti2_priors=relative_pose_priors, gt_wTi_list=gt_wTi_list + num_images, + pruned_i2Ri1_graph, + i1Ti2_priors=relative_pose_priors, + gt_wTi_list=gt_wTi_list, + v_corr_idxs=viewgraph_v_corr_idxs_graph, ) tracks2d_graph = dask.delayed(get_2d_tracks)(viewgraph_v_corr_idxs_graph, keypoints_list) diff --git a/gtsfm/two_view_estimator_cacher.py b/gtsfm/two_view_estimator_cacher.py index 3557631bd..a245948eb 100644 --- a/gtsfm/two_view_estimator_cacher.py +++ b/gtsfm/two_view_estimator_cacher.py @@ -38,6 +38,7 @@ class TwoViewEstimatorCacher(TwoViewEstimator): def __init__(self, two_view_estimator_obj: TwoViewEstimator) -> None: self._two_view_estimator = two_view_estimator_obj + self._verifier_key = self._two_view_estimator._verifier.__repr__() def __repr__(self) -> str: return self._two_view_estimator.__repr__() @@ -62,7 +63,8 @@ def __generate_cache_key( numpy_arrays_to_hash.append(keypoints_i2.coordinates[sampled_idxs[:, 1]].flatten()) # Hash the concatenation of all the numpy arrays. - return cache_utils.generate_hash_for_numpy_array(np.concatenate(numpy_arrays_to_hash)) + input_key = cache_utils.generate_hash_for_numpy_array(np.concatenate(numpy_arrays_to_hash)) + return f"{self._verifier_key}_{input_key}" def __load_result_from_cache( self, keypoints_i1: Keypoints, keypoints_i2: Keypoints, putative_corr_idxs: np.ndarray diff --git a/rtf_vis_tool/package-lock.json b/rtf_vis_tool/package-lock.json index c6183bf76..26ec825ec 100644 --- a/rtf_vis_tool/package-lock.json +++ b/rtf_vis_tool/package-lock.json @@ -5808,20 +5808,20 @@ "dev": true }, "node_modules/body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", - "raw-body": "2.5.1", + "qs": "6.11.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -6637,9 +6637,9 @@ } }, "node_modules/content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "engines": { "node": ">= 0.6" } @@ -6658,9 +6658,9 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -8782,16 +8782,16 @@ } }, "node_modules/express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -8807,7 +8807,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -9096,9 +9096,9 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -16257,9 +16257,9 @@ } }, "node_modules/qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "dependencies": { "side-channel": "^1.0.4" }, @@ -16395,9 +16395,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -19496,9 +19496,9 @@ } }, "node_modules/webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "dependencies": { "colorette": "^2.0.10", "memfs": "^3.4.3", @@ -24377,20 +24377,20 @@ "dev": true }, "body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", - "raw-body": "2.5.1", + "qs": "6.11.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -25037,9 +25037,9 @@ } }, "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "convert-source-map": { "version": "1.8.0", @@ -25057,9 +25057,9 @@ } }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -26635,16 +26635,16 @@ } }, "express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -26660,7 +26660,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -26898,9 +26898,9 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" }, "follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "for-each": { "version": "0.3.3", @@ -32072,9 +32072,9 @@ "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" }, "qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "requires": { "side-channel": "^1.0.4" } @@ -32168,9 +32168,9 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "requires": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -34547,9 +34547,9 @@ } }, "webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "requires": { "colorette": "^2.0.10", "memfs": "^3.4.3", diff --git a/scripts/benchmark_wildcat.sh b/scripts/benchmark_wildcat.sh index 12bd5f2ea..0da043052 100755 --- a/scripts/benchmark_wildcat.sh +++ b/scripts/benchmark_wildcat.sh @@ -7,218 +7,242 @@ now=$(date +"%Y%m%d_%H%M%S") datasets=( - # Tanks and Temples Dataset. - barn-tanks-and-temples-410 - # Olsson Datasets. - # See https://www.maths.lth.se/matematiklth/personal/calle/dataset/dataset.html - ecole-superieure-de-guerre-35 - fort-channing-gate-singapore-27 - skansen-kronan-gothenburg-131 - nijo-castle-gate-19 - kings-college-cambridge-328 - spilled-blood-cathedral-st-petersburg-781 - palace-fine-arts-281 - # 1dsfm Datasets - gendarmenmarkt-1463 - # Other. - skydio-crane-mast-501 - # Astrovision Datasets. - 2011205_rc3 - # Colmap Datasets. - south-building-128 - gerrard-hall-100 - ) + # Tanks and Temples Dataset. + barn-tanks-and-temples-410 + # Olsson Datasets. + # See https://www.maths.lth.se/matematiklth/personal/calle/dataset/dataset.html + ecole-superieure-de-guerre-35 + #fort-channing-gate-singapore-27 + skansen-kronan-gothenburg-131 + #nijo-castle-gate-19 + kings-college-cambridge-328 + spilled-blood-cathedral-st-petersburg-781 + palace-fine-arts-281 + # 1dsfm Datasets + # gendarmenmarkt-1463 + # Other. + skydio-crane-mast-501 + # Astrovision Datasets. + 2011205_rc3 + # Colmap Datasets. + south-building-128 + gerrard-hall-100 +) max_frame_lookahead_sizes=( - 0 - 5 - 10 - 15 - ) + # 0 + # 5 + 10 + #15 +) num_matched_sizes=( - 0 - 5 - 10 - 15 - 20 - 25 - ) + # 0 + 5 + # 10 + # 15 + # 20 + # 25 +) correspondence_generator_config_names=( - sift - lightglue - superglue - loftr - ) + sift + lightglue + superglue + loftr + disk +) if [[ $CLUSTER_CONFIG ]] then - CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" + CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" else - CLUSTER_ARGS="" + CLUSTER_ARGS="" fi for num_matched in ${num_matched_sizes[@]}; do - for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do - for dataset in ${datasets[@]}; do - if [[ $dataset == *"gendarmenmarkt-1463"* && $max_frame_lookahead != 0 ]] - then - # Gendarmenmarkt images have no natural order. - continue - fi - - if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] - then - # Matches must come from at least some retriever. - continue - fi - - for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do - - if [[ $correspondence_generator_config_name == *"sift"* ]] - then - num_workers=10 - elif [[ $correspondence_generator_config_name == *"lightglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"superglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"loftr"* ]] - then - num_workers=1 - fi - - echo "Dataset: ${dataset}" - echo "Num matched: ${num_matched}" - echo "Max frame lookahead: ${max_frame_lookahead}" - echo "Correspondence Generator: ${correspondence_generator_config_name}" - echo "Num workers: ${num_workers}" - - if [[ $dataset == *"barn-tanks-and-temples-410"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/Barn - colmap_files_dirpath=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/colmap_gt_2023_11_15_2250_highquality - elif [[ $dataset == *"palace-fine-arts-281"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/palace-fine-arts-281 - - elif [[ $dataset == *"ecole-superieure-de-guerre-35"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35 - - elif [[ $dataset == *"fort-channing-gate-singapore-27"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/fort-channing-gate-singapore-27 - - elif [[ $dataset == *"skansen-kronan-gothenburg-131"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131 - - elif [[ $dataset == *"nijo-castle-gate-19"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/nijo-castle-gate-19 - - elif [[ $dataset == *"kings-college-cambridge-328"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/kings-college-cambridge-328 - - elif [[ $dataset == *"spilled-blood-cathedral-st-petersburg-781"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781 - - elif [[ $dataset == *"skydio-crane-mast-501"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-crane-mast-501-images - colmap_files_dirpath=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-501-colmap-pseudo-gt - elif [[ $dataset == *"2011205_rc3"* ]] - then - loader=astrovision - data_dir=/usr/local/gtsfm-data/2011205_rc3 - elif [[ $dataset == *"south-building-128"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/south-building-128/images - colmap_files_dirpath=/usr/local/gtsfm-data/south-building-128/colmap-2023-07-28-txt - elif [[ $dataset == *"gerrard-hall-100"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/gerrard-hall-100/images - colmap_files_dirpath=/usr/local/gtsfm-data/gerrard-hall-100/colmap-3.7-sparse-txt-2023-07-27 - elif [[ $dataset == *"gendarmenmarkt-1463"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/Gendarmenmarkt/images - colmap_files_dirpath=/usr/local/gtsfm-data/Gendarmenmarkt/gendarmenmark/im_size_full/0 - fi - - OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} - mkdir -p $OUTPUT_ROOT - - if [[ $loader == *"olsson"* ]] - then - python gtsfm/runner/run_scene_optimizer_olssonloader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --dataset_root $dataset_root \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - elif [[ $loader == *"colmap"* ]] - then - python gtsfm/runner/run_scene_optimizer_colmaploader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --images_dir $images_dir \ - --colmap_files_dirpath $colmap_files_dirpath \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - elif [[ $loader == *"astrovision"* ]] - then - python gtsfm/runner/run_scene_optimizer_astrovision.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --data_dir $data_dir \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - fi - done - done - done + for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do + for dataset in ${datasets[@]}; do + if [[ $dataset == *"gendarmenmarkt-1463"* && $max_frame_lookahead != 0 ]] + then + # Gendarmenmarkt images have no natural order. + continue + fi + + if [[ $dataset == *"gendarmenmarkt-1463"* ]] + then + INTRINSICS_ARGS="" + else + INTRINSICS_ARGS="--share_intrinsics" + fi + + if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] + then + # Matches must come from at least some retriever. + continue + fi + + for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do + + if [[ $correspondence_generator_config_name == *"sift"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"lightglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"superglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"disk"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"loftr"* ]] + then + num_workers=1 + fi + + echo "Dataset: ${dataset}" + echo "Num matched: ${num_matched}" + echo "Max frame lookahead: ${max_frame_lookahead}" + echo "Correspondence Generator: ${correspondence_generator_config_name}" + echo "Num workers: ${num_workers}" + echo "Intrinsics: ${INTRINSICS_ARGS}" + + if [[ $dataset == *"barn-tanks-and-temples-410"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/Barn + colmap_files_dirpath=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/colmap_gt_2023_11_15_2250_highquality + + elif [[ $dataset == *"palace-fine-arts-281"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/palace-fine-arts-281 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/palace_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/palace-fine-arts-281/images + + elif [[ $dataset == *"ecole-superieure-de-guerre-35"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/ecole_superieure_colmap_gt_2023_11_22/txt_gt + images_dir=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35/images + + elif [[ $dataset == *"fort-channing-gate-singapore-27"* ]] + then + loader=olsson + dataset_root=/usr/local/gtsfm-data/fort-channing-gate-singapore-27 + + elif [[ $dataset == *"skansen-kronan-gothenburg-131"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/skansen_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131/images + + elif [[ $dataset == *"nijo-castle-gate-19"* ]] + then + loader=olsson + dataset_root=/usr/local/gtsfm-data/nijo-castle-gate-19 + + elif [[ $dataset == *"kings-college-cambridge-328"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/kings-college-cambridge-328 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/kings_college_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/kings-college-cambridge-328/images + + elif [[ $dataset == *"spilled-blood-cathedral-st-petersburg-781"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/colmap-spilled-blood-gt + images_dir=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781/images + + elif [[ $dataset == *"skydio-crane-mast-501"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-crane-mast-501-images + colmap_files_dirpath=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-501-colmap-pseudo-gt + elif [[ $dataset == *"2011205_rc3"* ]] + then + loader=astrovision + data_dir=/usr/local/gtsfm-data/2011205_rc3 + elif [[ $dataset == *"south-building-128"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/south-building-128/images + colmap_files_dirpath=/usr/local/gtsfm-data/south-building-128/colmap-2023-07-28-txt + elif [[ $dataset == *"gerrard-hall-100"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/gerrard-hall-100/images + colmap_files_dirpath=/usr/local/gtsfm-data/gerrard-hall-100/colmap-3.7-sparse-txt-2023-07-27 + elif [[ $dataset == *"gendarmenmarkt-1463"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/Gendarmenmarkt/images + colmap_files_dirpath=/usr/local/gtsfm-data/Gendarmenmarkt/gendarmenmark/im_size_full/0 + fi + + OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} + mkdir -p $OUTPUT_ROOT + + if [[ $loader == *"olsson"* ]] + then + python gtsfm/runner/run_scene_optimizer_olssonloader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --dataset_root $dataset_root \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + elif [[ $loader == *"colmap"* ]] + then + python gtsfm/runner/run_scene_optimizer_colmaploader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --images_dir $images_dir \ + --colmap_files_dirpath $colmap_files_dirpath \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + elif [[ $loader == *"astrovision"* ]] + then + python gtsfm/runner/run_scene_optimizer_astrovision.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --data_dir $data_dir \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + fi + done + done + done done - diff --git a/scripts/eth3d_benchmark.sh b/scripts/eth3d_benchmark.sh index e6c6f9afe..32ba6543c 100755 --- a/scripts/eth3d_benchmark.sh +++ b/scripts/eth3d_benchmark.sh @@ -6,112 +6,118 @@ CLUSTER_CONFIG=$2 now=$(date +"%Y%m%d_%H%M%S") -ETH3D_ROOT=/home/tdriver6/Downloads/eth3d_datasets +ETH3D_ROOT=/usr/local/gtsfm-data/eth3d_datasets/multi_view_training_dslr_undistorted +#ETH3D_ROOT=/home/tdriver6/Downloads/eth3d_datasets # Includes all "high-resolution multi-view" datasets from 'training' split (i.e. w/ public GT data) # See https://www.eth3d.net/datasets for more information. datasets=( - courtyard - delivery_area - electro - facade - kicker - meadow - office - pipes - playground - relief_2 - relief - terrace - terrains - ) + courtyard + delivery_area + electro + facade + kicker + meadow + office + pipes + playground + relief_2 + relief + terrace + terrains +) max_frame_lookahead_sizes=( - #0 - #5 - 10 - #15 - ) + #0 + #5 + 10 + #15 +) num_matched_sizes=( - 5 - #10 - #15 - #20 - #25 - ) + 5 + #10 + #15 + #20 + #25 +) correspondence_generator_config_names=( - sift - #disk - #lightglue - #superglue - #loftr - ) + sift + disk + lightglue + superglue + loftr +) if [[ $CLUSTER_CONFIG ]] then - CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" + CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" else - CLUSTER_ARGS="" + CLUSTER_ARGS="" fi for num_matched in ${num_matched_sizes[@]}; do - for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do - for dataset in ${datasets[@]}; do - if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] - then - # Matches must come from at least some retriever. - continue - fi - - for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do - - if [[ $correspondence_generator_config_name == *"sift"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"lightglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"superglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"loftr"* ]] - then - num_workers=1 - fi - - echo "Dataset: ${dataset}" - echo "Num matched: ${num_matched}" - echo "Max frame lookahead: ${max_frame_lookahead}" - echo "Correspondence Generator: ${correspondence_generator_config_name}" - echo "Num workers: ${num_workers}" - - images_dir="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/images" - colmap_files_dirpath="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/dslr_calibration_undistorted" - - OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} - mkdir -p $OUTPUT_ROOT - - python gtsfm/runner/run_scene_optimizer_colmaploader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --images_dir $images_dir \ - --colmap_files_dirpath $colmap_files_dirpath \ - --num_workers 1 \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - done - done - done + for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do + for dataset in ${datasets[@]}; do + if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] + then + # Matches must come from at least some retriever. + continue + fi + + for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do + + if [[ $correspondence_generator_config_name == *"sift"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"lightglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"superglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"loftr"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"disk"* ]] + then + num_workers=1 + fi + + echo "Dataset: ${dataset}" + echo "Num matched: ${num_matched}" + echo "Max frame lookahead: ${max_frame_lookahead}" + echo "Correspondence Generator: ${correspondence_generator_config_name}" + echo "Num workers: ${num_workers}" + + images_dir="${ETH3D_ROOT}/${dataset}/images" + colmap_files_dirpath="${ETH3D_ROOT}/${dataset}/dslr_calibration_undistorted" + # images_dir="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/images" + # colmap_files_dirpath="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/dslr_calibration_undistorted" + + OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} + mkdir -p $OUTPUT_ROOT + + python gtsfm/runner/run_scene_optimizer_colmaploader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --share_intrinsics \ + --images_dir $images_dir \ + --colmap_files_dirpath $colmap_files_dirpath \ + --num_workers 1 \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + done + done + done done diff --git a/tests/averaging/rotation/test_shonan.py b/tests/averaging/rotation/test_shonan.py index b0aef22a4..20f40be64 100644 --- a/tests/averaging/rotation/test_shonan.py +++ b/tests/averaging/rotation/test_shonan.py @@ -38,7 +38,8 @@ def __execute_test(self, i2Ri1_input: Dict[Tuple[int, int], Rot3], wRi_expected: wRi_expected: expected global rotations. """ i1Ti2_priors: Dict[Tuple[int, int], PosePrior] = {} - wRi_computed = self.obj.run_rotation_averaging(len(wRi_expected), i2Ri1_input, i1Ti2_priors) + v_corr_idxs = _create_dummy_correspondences(i2Ri1_input) + wRi_computed = self.obj.run_rotation_averaging(len(wRi_expected), i2Ri1_input, i1Ti2_priors, v_corr_idxs) self.assertTrue( geometry_comparisons.compare_rotations(wRi_computed, wRi_expected, ROTATION_ANGLE_ERROR_THRESHOLD_DEG) ) @@ -103,8 +104,8 @@ def test_simple_with_prior(self): type=PosePriorType.SOFT_CONSTRAINT, ) } - - wRi_computed = self.obj.run_rotation_averaging(len(expected_wRi_list), i2Ri1_dict, i1Ti2_priors) + v_corr_idxs = _create_dummy_correspondences(i2Ri1_dict) + wRi_computed = self.obj.run_rotation_averaging(len(expected_wRi_list), i2Ri1_dict, i1Ti2_priors, v_corr_idxs) self.assertTrue( geometry_comparisons.compare_rotations(wRi_computed, expected_wRi_list, ROTATION_ANGLE_ERROR_THRESHOLD_DEG) ) @@ -121,13 +122,16 @@ def test_computation_graph(self): i2Ri1_graph = dask.delayed(i2Ri1_dict) - # use the GTSAM API directly (without dask) for rotation averaging + # Use the GTSAM API directly (without dask) for rotation averaging i1Ti2_priors: Dict[Tuple[int, int], PosePrior] = {} - expected_wRi_list = self.obj.run_rotation_averaging(num_poses, i2Ri1_dict, i1Ti2_priors) + v_corr_idxs = _create_dummy_correspondences(i2Ri1_dict) + expected_wRi_list = self.obj.run_rotation_averaging(num_poses, i2Ri1_dict, i1Ti2_priors, v_corr_idxs) - # use dask's computation graph + # Use dask's computation graph gt_wTi_list = [None] * len(expected_wRi_list) - rotations_graph, _ = self.obj.create_computation_graph(num_poses, i2Ri1_graph, i1Ti2_priors, gt_wTi_list) + rotations_graph, _ = self.obj.create_computation_graph( + num_poses, i2Ri1_graph, i1Ti2_priors, v_corr_idxs=v_corr_idxs, gt_wTi_list=gt_wTi_list + ) with dask.config.set(scheduler="single-threaded"): wRi_list = dask.compute(rotations_graph)[0] @@ -168,12 +172,23 @@ def test_nonconsecutive_indices(self): } relative_pose_priors: Dict[Tuple[int, int], PosePrior] = {} - wRi_computed = self.obj.run_rotation_averaging(num_images, i2Ri1_input, relative_pose_priors) + v_corr_idxs = _create_dummy_correspondences(i2Ri1_input) + wRi_computed = self.obj.run_rotation_averaging(num_images, i2Ri1_input, relative_pose_priors, v_corr_idxs) wRi_expected = [None, wTi1.rotation(), wTi2.rotation(), wTi3.rotation()] self.assertTrue( geometry_comparisons.compare_rotations(wRi_computed, wRi_expected, angular_error_threshold_degrees=0.1) ) +def _create_dummy_correspondences(i2Ri1_dict: Dict[Tuple[int, int], Rot3]) -> Dict[Tuple[int, int], np.ndarray]: + """Create dummy verified correspondences for each edge in view graph.""" + # Assume image has shape (img_h, img_w) = (1000,1000) + img_h = 1000 + v_corr_idxs_dict = { + (i1, i2): np.random.randint(low=0, high=img_h, size=(i1 + i2, 2)) for i1, i2 in i2Ri1_dict.keys() + } + return v_corr_idxs_dict + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_two_view_estimator_cacher.py b/tests/test_two_view_estimator_cacher.py index c95be44c0..386b0310f 100644 --- a/tests/test_two_view_estimator_cacher.py +++ b/tests/test_two_view_estimator_cacher.py @@ -45,6 +45,12 @@ def test_cache_miss( # Mock the underlying two-view estimator which is used on cache miss. underlying_estimator_mock = MagicMock() underlying_estimator_mock.run_2view.return_value = self.dummy_output + verifier_key = "Ransac__use_intrinsicsTrue_4px" + + def new_repr(self) -> str: + return verifier_key + + underlying_estimator_mock._verifier.__repr__ = new_repr cacher = TwoViewEstimatorCacher(two_view_estimator_obj=underlying_estimator_mock) @@ -69,11 +75,10 @@ def test_cache_miss( generate_hash_for_numpy_array_mock.assert_called() # Assert that read function was called once and write function was called once. - cache_path = ROOT_PATH / "cache" / "two_view_estimator" / "numpy_key.pbz2" + cache_path = ROOT_PATH / "cache" / "two_view_estimator" / f"{verifier_key}_numpy_key.pbz2" read_mock.assert_called_once_with(cache_path) write_mock.assert_called_once() - @patch("gtsfm.utils.cache.generate_hash_for_numpy_array", return_value="numpy_key") @patch("gtsfm.utils.io.read_from_bz2_file", return_value=_DUMMY_OUTPUT) @patch("gtsfm.utils.io.write_to_bz2_file") @@ -89,7 +94,12 @@ def test_cache_hit( # Mock the underlying two-view estimator which is used on cache miss. underlying_estimator_mock = MagicMock() underlying_estimator_mock.run_2view.return_value = self.dummy_output + verifier_key = "Ransac__use_intrinsicsTrue_4px" + + def new_repr(self) -> str: + return verifier_key + underlying_estimator_mock._verifier.__repr__ = new_repr cacher = TwoViewEstimatorCacher(two_view_estimator_obj=underlying_estimator_mock) result = cacher.run_2view( @@ -114,7 +124,7 @@ def test_cache_hit( generate_hash_for_numpy_array_mock.assert_called() # Assert that the read function was called once. - cache_path = ROOT_PATH / "cache" / "two_view_estimator" / "numpy_key.pbz2" + cache_path = ROOT_PATH / "cache" / "two_view_estimator" / f"{verifier_key}_numpy_key.pbz2" read_mock.assert_called_once_with(cache_path) # Assert that the write function was not called (as cache is mocked to already exist).