[Refactor] Refactoring the code of Indoor Metric with MMEval

.circleci/test.yml

@@ -61,6 +61,7 @@ jobs:
           name: Install mmdet3d dependencies
           command: |
             python -m pip install git+ssh://[email protected]/open-mmlab/mmengine.git@main
+            python -m pip install git+ssh://[email protected]/open-mmlab/mmeval.git@main
             python -m pip install -U openmim
             python -m mim install 'mmcv >= 2.0.0rc1'
             python -m pip install git+ssh://[email protected]/open-mmlab/[email protected]
@@ -96,16 +97,18 @@ jobs:
           name: Clone Repos
           command: |
             git clone -b main --depth 1 ssh://[email protected]/open-mmlab/mmengine.git /home/circleci/mmengine
+            git clone -b main --depth 1 ssh://[email protected]/open-mmlab/mmeval.git /home/circleci/mmeval
             git clone -b dev-3.x --depth 1 ssh://[email protected]/open-mmlab/mmdetection.git /home/circleci/mmdetection
       - run:
           name: Build Docker image
           command: |
             docker build .circleci/docker -t mmdet3d:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >>
-            docker run --gpus all -t -d -v /home/circleci/project:/mmdetection3d -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmdetection:/mmdetection -w /mmdetection3d --name mmdet3d mmdet3d:gpu
+            docker run --gpus all -t -d -v /home/circleci/project:/mmdetection3d -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmeval:/mmeval -v /home/circleci/mmdetection:/mmdetection -w /mmdetection3d --name mmdet3d mmdet3d:gpu
       - run:
           name: Install mmdet3d dependencies
           command: |
             docker exec mmdet3d pip install -e /mmengine
+            docker exec mmdet3d pip install -e /mmeval
             docker exec mmdet3d pip install -U openmim
             docker exec mmdet3d mim install 'mmcv >= 2.0.0rc1'
             docker exec mmdet3d pip install -e /mmdetection

mmdet3d/evaluation/metrics/indoor_metric.py

@@ -6,36 +6,27 @@
 from mmdet.evaluation import eval_map
 from mmengine.evaluator import BaseMetric
 from mmengine.logging import MMLogger
+from mmeval import Indoor3DMeanAP
 
-from mmdet3d.evaluation import indoor_eval
 from mmdet3d.registry import METRICS
-from mmdet3d.structures import get_box_type
 
 
 @METRICS.register_module()
-class IndoorMetric(BaseMetric):
+class IndoorMetric(Indoor3DMeanAP):
     """Indoor scene evaluation metric.
 
     Args:
         iou_thr (float or List[float]): List of iou threshold when calculate
             the metric. Defaults to [0.25, 0.5].
-        collect_device (str): Device name used for collecting results from
-            different ranks during distributed training. Must be 'cpu' or
-            'gpu'. Defaults to 'cpu'.
-        prefix (str, optional): The prefix that will be added in the metric
-            names to disambiguate homonymous metrics of different evaluators.
-            If prefix is not provided in the argument, self.default_prefix will
-            be used instead. Defaults to None.
+        **kwargs: Keyword parameters passed to :class:`BaseMetric`.
     """
 
-    def __init__(self,
-                 iou_thr: List[float] = [0.25, 0.5],
-                 collect_device: str = 'cpu',
-                 prefix: Optional[str] = None) -> None:
+    def __init__(self, iou_thr: List[float] = [0.25, 0.5], **kwargs) -> None:
+        logger: MMLogger = MMLogger.get_current_instance()
         super(IndoorMetric, self).__init__(
-            prefix=prefix, collect_device=collect_device)
-        self.iou_thr = [iou_thr] if isinstance(iou_thr, float) else iou_thr
+            iou_thr=iou_thr, logger=logger, **kwargs)
 
+    # TODO: remove data_batch
     def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
         """Process one batch of data samples and predictions.
 
@@ -46,48 +37,30 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None:
             data_batch (dict): A batch of data from the dataloader.
             data_samples (Sequence[dict]): A batch of outputs from the model.
         """
+        predictions, groundtruths = [], []
         for data_sample in data_samples:
-            pred_3d = data_sample['pred_instances_3d']
-            eval_ann_info = data_sample['eval_ann_info']
-            cpu_pred_3d = dict()
-            for k, v in pred_3d.items():
-                if hasattr(v, 'to'):
-                    cpu_pred_3d[k] = v.to('cpu')
-                else:
-                    cpu_pred_3d[k] = v
-            self.results.append((eval_ann_info, cpu_pred_3d))
-
-    def compute_metrics(self, results: list) -> Dict[str, float]:
-        """Compute the metrics from processed results.
 
-        Args:
-            results (list): The processed results of each batch.
-
-        Returns:
-            Dict[str, float]: The computed metrics. The keys are the names of
-            the metrics, and the values are corresponding results.
+            groundtruth = data_sample['eval_ann_info']
+            groundtruths.append(groundtruth)
+            prediction = dict()
+            prediction['scores_3d'] = data_sample['pred_instances_3d'][
+                'scores_3d'].cpu().numpy()
+            prediction['labels_3d'] = data_sample['pred_instances_3d'][
+                'labels_3d'].cpu().numpy()
+            prediction['bboxes_3d'] = data_sample['pred_instances_3d'][
+                'bboxes_3d'].to('cpu')
+            predictions.append(prediction)
+        self.add(predictions, groundtruths)
+
+    def evaluate(self, *args, **kwargs) -> dict:
+        """Returns metric results and print pretty table of metrics per class.
+
+        This method would be invoked by ``mmengine.Evaluator``. After
+        refactoring we do not return less readable information.
         """
-        logger: MMLogger = MMLogger.get_current_instance()
-        ann_infos = []
-        pred_results = []
-
-        for eval_ann, sinlge_pred_results in results:
-            ann_infos.append(eval_ann)
-            pred_results.append(sinlge_pred_results)
-
-        # some checkpoints may not record the key "box_type_3d"
-        box_type_3d, box_mode_3d = get_box_type(
-            self.dataset_meta.get('box_type_3d', 'depth'))
-
-        ret_dict = indoor_eval(
-            ann_infos,
-            pred_results,
-            self.iou_thr,
-            self.dataset_meta['classes'],
-            logger=logger,
-            box_mode_3d=box_mode_3d)
-
-        return ret_dict
+        metric_results = self.compute(*args, **kwargs)
+        self.reset()
+        return metric_results
 
 
 @METRICS.register_module()

mmdet3d/models/layers/pointnet_modules/builder.py

@@ -4,7 +4,8 @@
 from mmengine.registry import Registry
 from torch import nn as nn
 
-SA_MODULES = Registry('point_sa_module')
+SA_MODULES = Registry(
+    'point_sa_module', locations=['mmdet3d.models.layers.pointnet_modules'])
 
 
 def build_sa_module(cfg: Union[dict, None], *args, **kwargs) -> nn.Module:

requirements/mminstall.txt

@@ -1,3 +1,4 @@
 mmcv>=2.0.0rc4,<2.1.0
 mmdet>=3.0.0rc0,<3.1.0
 mmengine>=0.6.0,<1.0.0
+mmeval