diff --git a/.circleci/test.yml b/.circleci/test.yml index e8e3498b2d..7df1c1b4ad 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -61,6 +61,7 @@ jobs: name: Install mmdet3d dependencies command: | python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main + python -m pip install git+ssh://git@github.com/open-mmlab/mmeval.git@main python -m pip install -U openmim python -m mim install 'mmcv >= 2.0.0rc1' python -m pip install git+ssh://git@github.com/open-mmlab/mmdetection.git@dev-3.x @@ -96,16 +97,18 @@ jobs: name: Clone Repos command: | git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine + git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmeval.git /home/circleci/mmeval git clone -b dev-3.x --depth 1 ssh://git@github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection - run: name: Build Docker image command: | docker build .circleci/docker -t mmdet3d:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> - docker run --gpus all -t -d -v /home/circleci/project:/mmdetection3d -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmdetection:/mmdetection -w /mmdetection3d --name mmdet3d mmdet3d:gpu + docker run --gpus all -t -d -v /home/circleci/project:/mmdetection3d -v /home/circleci/mmengine:/mmengine -v /home/circleci/mmeval:/mmeval -v /home/circleci/mmdetection:/mmdetection -w /mmdetection3d --name mmdet3d mmdet3d:gpu - run: name: Install mmdet3d dependencies command: | docker exec mmdet3d pip install -e /mmengine + docker exec mmdet3d pip install -e /mmeval docker exec mmdet3d pip install -U openmim docker exec mmdet3d mim install 'mmcv >= 2.0.0rc1' docker exec mmdet3d pip install -e /mmdetection diff --git a/mmdet3d/evaluation/metrics/indoor_metric.py b/mmdet3d/evaluation/metrics/indoor_metric.py index 987e641a33..8f8b2bdebf 100644 --- a/mmdet3d/evaluation/metrics/indoor_metric.py +++ b/mmdet3d/evaluation/metrics/indoor_metric.py @@ -6,36 +6,27 @@ from mmdet.evaluation import eval_map from mmengine.evaluator import BaseMetric from mmengine.logging import MMLogger +from mmeval import Indoor3DMeanAP -from mmdet3d.evaluation import indoor_eval from mmdet3d.registry import METRICS -from mmdet3d.structures import get_box_type @METRICS.register_module() -class IndoorMetric(BaseMetric): +class IndoorMetric(Indoor3DMeanAP): """Indoor scene evaluation metric. Args: iou_thr (float or List[float]): List of iou threshold when calculate the metric. Defaults to [0.25, 0.5]. - collect_device (str): Device name used for collecting results from - different ranks during distributed training. Must be 'cpu' or - 'gpu'. Defaults to 'cpu'. - prefix (str, optional): The prefix that will be added in the metric - names to disambiguate homonymous metrics of different evaluators. - If prefix is not provided in the argument, self.default_prefix will - be used instead. Defaults to None. + **kwargs: Keyword parameters passed to :class:`BaseMetric`. """ - def __init__(self, - iou_thr: List[float] = [0.25, 0.5], - collect_device: str = 'cpu', - prefix: Optional[str] = None) -> None: + def __init__(self, iou_thr: List[float] = [0.25, 0.5], **kwargs) -> None: + logger: MMLogger = MMLogger.get_current_instance() super(IndoorMetric, self).__init__( - prefix=prefix, collect_device=collect_device) - self.iou_thr = [iou_thr] if isinstance(iou_thr, float) else iou_thr + iou_thr=iou_thr, logger=logger, **kwargs) + # TODO: remove data_batch def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. @@ -46,48 +37,30 @@ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of outputs from the model. """ + predictions, groundtruths = [], [] for data_sample in data_samples: - pred_3d = data_sample['pred_instances_3d'] - eval_ann_info = data_sample['eval_ann_info'] - cpu_pred_3d = dict() - for k, v in pred_3d.items(): - if hasattr(v, 'to'): - cpu_pred_3d[k] = v.to('cpu') - else: - cpu_pred_3d[k] = v - self.results.append((eval_ann_info, cpu_pred_3d)) - - def compute_metrics(self, results: list) -> Dict[str, float]: - """Compute the metrics from processed results. - Args: - results (list): The processed results of each batch. - - Returns: - Dict[str, float]: The computed metrics. The keys are the names of - the metrics, and the values are corresponding results. + groundtruth = data_sample['eval_ann_info'] + groundtruths.append(groundtruth) + prediction = dict() + prediction['scores_3d'] = data_sample['pred_instances_3d'][ + 'scores_3d'].cpu().numpy() + prediction['labels_3d'] = data_sample['pred_instances_3d'][ + 'labels_3d'].cpu().numpy() + prediction['bboxes_3d'] = data_sample['pred_instances_3d'][ + 'bboxes_3d'].to('cpu') + predictions.append(prediction) + self.add(predictions, groundtruths) + + def evaluate(self, *args, **kwargs) -> dict: + """Returns metric results and print pretty table of metrics per class. + + This method would be invoked by ``mmengine.Evaluator``. After + refactoring we do not return less readable information. """ - logger: MMLogger = MMLogger.get_current_instance() - ann_infos = [] - pred_results = [] - - for eval_ann, sinlge_pred_results in results: - ann_infos.append(eval_ann) - pred_results.append(sinlge_pred_results) - - # some checkpoints may not record the key "box_type_3d" - box_type_3d, box_mode_3d = get_box_type( - self.dataset_meta.get('box_type_3d', 'depth')) - - ret_dict = indoor_eval( - ann_infos, - pred_results, - self.iou_thr, - self.dataset_meta['classes'], - logger=logger, - box_mode_3d=box_mode_3d) - - return ret_dict + metric_results = self.compute(*args, **kwargs) + self.reset() + return metric_results @METRICS.register_module() diff --git a/mmdet3d/models/layers/pointnet_modules/builder.py b/mmdet3d/models/layers/pointnet_modules/builder.py index 617a8942e6..7361aa3e21 100644 --- a/mmdet3d/models/layers/pointnet_modules/builder.py +++ b/mmdet3d/models/layers/pointnet_modules/builder.py @@ -4,7 +4,8 @@ from mmengine.registry import Registry from torch import nn as nn -SA_MODULES = Registry('point_sa_module') +SA_MODULES = Registry( + 'point_sa_module', locations=['mmdet3d.models.layers.pointnet_modules']) def build_sa_module(cfg: Union[dict, None], *args, **kwargs) -> nn.Module: diff --git a/requirements/mminstall.txt b/requirements/mminstall.txt index 0f8afd34c1..e00f049b37 100644 --- a/requirements/mminstall.txt +++ b/requirements/mminstall.txt @@ -1,3 +1,4 @@ mmcv>=2.0.0rc4,<2.1.0 mmdet>=3.0.0rc0,<3.1.0 mmengine>=0.6.0,<1.0.0 +mmeval