Skip to content

Commit 4306849

Browse files
BIGWangYuDongice-tongzhouzaida
authored
[Feature] Add COCO Metric (#23)
* [Feature] Add COCO Metric * add pycocotools as an optional requirement * minor fix * Update mmeval/metrics/coco.py Co-authored-by: yancong <[email protected]> * Update mmeval/metrics/coco.py Co-authored-by: yancong <[email protected]> * minor fix * Add UT * update CI config * update comments * update comments * rename * delete mmengine * update ci * minor fix * rename * Update mmeval/metrics/coco_detection.py Co-authored-by: Zaida Zhou <[email protected]> * Update mmeval/metrics/coco_detection.py Co-authored-by: Zaida Zhou <[email protected]> * Update mmeval/metrics/coco_detection.py Co-authored-by: Zaida Zhou <[email protected]> * Update mmeval/metrics/coco_detection.py Co-authored-by: Zaida Zhou <[email protected]> * minor fix * minor fix * minor fix * minor fix Co-authored-by: yancong <[email protected]> Co-authored-by: Zaida Zhou <[email protected]>
1 parent c4bc380 commit 4306849

File tree

9 files changed

+1716
-15
lines changed

9 files changed

+1716
-15
lines changed

mmeval/metrics/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Copyright (c) OpenMMLab. All rights reserved.
22

33
from .accuracy import Accuracy
4+
from .coco_detection import COCODetectionMetric
45
from .end_point_error import EndPointError
56
from .f_metric import F1Metric
67
from .hmean_iou import HmeanIoU
@@ -10,5 +11,5 @@
1011

1112
__all__ = [
1213
'Accuracy', 'MeanIoU', 'VOCMeanAP', 'OIDMeanAP', 'EndPointError',
13-
'F1Metric', 'HmeanIoU'
14+
'F1Metric', 'HmeanIoU', 'COCODetectionMetric'
1415
]

mmeval/metrics/coco_detection.py

Lines changed: 618 additions & 0 deletions
Large diffs are not rendered by default.

mmeval/metrics/utils/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
# Copyright (c) OpenMMLab. All rights reserved.
2-
32
from .hmean import compute_hmean
43
from .polygon import (poly2shapely, poly_intersection, poly_iou,
54
poly_make_valid, poly_union, polys2shapely)
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
# Copyright (c) OpenMMLab. All rights reserved.
2+
import pycocotools.mask as _mask_util
3+
from collections import defaultdict
4+
from pathlib import Path
5+
from pycocotools.coco import COCO as _COCO
6+
from pycocotools.cocoeval import COCOeval as _COCOeval
7+
from typing import Dict, Optional, Sequence, Union
8+
9+
10+
class COCO(_COCO):
11+
"""This class is almost the same as official pycocotools package.
12+
13+
It implements some snake case function aliases. So that the COCO class has
14+
the same interface as LVIS class.
15+
16+
Args:
17+
annotation_file (str, optional): Path of annotation file.
18+
Defaults to None.
19+
"""
20+
21+
def __init__(self,
22+
annotation_file: Optional[Union[str, Path]] = None) -> None:
23+
super().__init__(annotation_file=annotation_file)
24+
self.img_ann_map = self.imgToAnns
25+
self.cat_img_map = self.catToImgs
26+
27+
def get_ann_ids(self,
28+
img_ids: Union[list, int] = [],
29+
cat_ids: Union[list, int] = [],
30+
area_rng: Union[list, int] = [],
31+
iscrowd: Optional[bool] = None) -> list:
32+
"""Get annotation ids that satisfy given filter conditions.
33+
34+
Args:
35+
img_ids (list | int): Get annotations for given images.
36+
cat_ids (list | int): Get categories for given images.
37+
area_rng (list | int): Get annotations for given area range.
38+
iscrowd (bool, optional): Get annotations for given crowd label.
39+
40+
Returns:
41+
List: Integer array of annotation ids.
42+
"""
43+
return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd)
44+
45+
def get_cat_ids(self,
46+
cat_names: Union[list, int] = [],
47+
sup_names: Union[list, int] = [],
48+
cat_ids: Union[list, int] = []) -> list:
49+
"""Get category ids that satisfy given filter conditions.
50+
51+
Args:
52+
cat_names (list | int): Get categories for given category names.
53+
sup_names (list | int): Get categories for given supercategory
54+
names.
55+
cat_ids (list | int): Get categories for given category ids.
56+
57+
Returns:
58+
List: Integer array of category ids.
59+
"""
60+
return self.getCatIds(cat_names, sup_names, cat_ids)
61+
62+
def get_img_ids(self,
63+
img_ids: Union[list, int] = [],
64+
cat_ids: Union[list, int] = []) -> list:
65+
"""Get image ids that satisfy given filter conditions.
66+
67+
Args:
68+
img_ids (list | int): Get images for given ids
69+
cat_ids (list | int): Get images with all given cats
70+
71+
Returns:
72+
List: Integer array of image ids.
73+
"""
74+
return self.getImgIds(img_ids, cat_ids)
75+
76+
def load_anns(self, ids: Union[list, int] = []) -> list:
77+
"""Load annotations with the specified ids.
78+
79+
Args:
80+
ids (list | int): Integer ids specifying annotations.
81+
82+
Returns:
83+
List[dict]: Loaded annotation objects.
84+
"""
85+
return self.loadAnns(ids)
86+
87+
def load_cats(self, ids: Union[list, int] = []) -> list:
88+
"""Load categories with the specified ids.
89+
90+
Args:
91+
ids (list | int): Integer ids specifying categories.
92+
93+
Returns:
94+
List[dict]: loaded category objects.
95+
"""
96+
return self.loadCats(ids)
97+
98+
def load_imgs(self, ids: Union[list, int] = []) -> list:
99+
"""Load annotations with the specified ids.
100+
101+
Args:
102+
ids (list): integer ids specifying image.
103+
104+
Returns:
105+
List[dict]: Loaded image objects.
106+
"""
107+
return self.loadImgs(ids)
108+
109+
110+
class COCOPanoptic(COCO):
111+
"""This wrapper is for loading the panoptic style annotation file."""
112+
113+
def createIndex(self) -> None:
114+
"""Create index."""
115+
# create index
116+
print('creating index...')
117+
# anns stores 'segment_id -> annotation'
118+
anns: Dict[int, list] = {}
119+
cats: Dict[int, dict] = {}
120+
imgs: Dict[int, dict] = {}
121+
img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list)
122+
if 'annotations' in self.dataset:
123+
for ann in self.dataset['annotations']:
124+
for seg_ann in ann['segments_info']:
125+
# to match with instance.json
126+
seg_ann['image_id'] = ann['image_id']
127+
img_to_anns[ann['image_id']].append(seg_ann)
128+
# segment_id is not unique in coco dataset orz...
129+
# annotations from different images but
130+
# may have same segment_id
131+
if seg_ann['id'] in anns.keys():
132+
anns[seg_ann['id']].append(seg_ann)
133+
else:
134+
anns[seg_ann['id']] = [seg_ann]
135+
136+
# filter out annotations from other images
137+
img_to_anns_ = defaultdict(list)
138+
for k, v in img_to_anns.items():
139+
img_to_anns_[k] = [x for x in v if x['image_id'] == k]
140+
img_to_anns = img_to_anns_
141+
142+
if 'images' in self.dataset:
143+
for img_info in self.dataset['images']:
144+
img_info['segm_file'] = img_info['file_name'].replace(
145+
'jpg', 'png')
146+
imgs[img_info['id']] = img_info
147+
148+
if 'categories' in self.dataset:
149+
for cat in self.dataset['categories']:
150+
cats[cat['id']] = cat
151+
152+
if 'annotations' in self.dataset and 'categories' in self.dataset:
153+
for ann in self.dataset['annotations']:
154+
for seg_ann in ann['segments_info']:
155+
cat_to_imgs[seg_ann['category_id']].append(ann['image_id'])
156+
157+
print('index created!')
158+
159+
self.anns = anns
160+
self.imgToAnns = img_to_anns
161+
self.catToImgs = cat_to_imgs
162+
self.imgs = imgs
163+
self.cats = cats
164+
165+
def load_anns(self, ids: Union[list, int] = []) -> list:
166+
"""Load annotations with the specified ids.
167+
168+
``self.anns`` is a list of annotation lists instead of a
169+
list of annotations.
170+
171+
Args:
172+
ids (Union[List[int], int]): Integer ids specifying annotations.
173+
174+
Returns:
175+
List: Loaded annotation objects.
176+
"""
177+
anns = []
178+
179+
if isinstance(ids, Sequence):
180+
# self.anns is a list of annotation lists instead of
181+
# a list of annotations
182+
for id in ids:
183+
anns += self.anns[id]
184+
return anns
185+
else:
186+
return self.anns[ids]
187+
188+
189+
# just for the ease of import
190+
COCOeval = _COCOeval
191+
mask_util = _mask_util

mmeval/metrics/voc_map.py

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from typing import Dict, List, Optional, Sequence, Tuple, Union
55

66
from mmeval.core.base_metric import BaseMetric
7+
from mmeval.utils import is_list_of
78

89

910
def calculate_average_precision(recalls: np.ndarray,
@@ -199,9 +200,8 @@ class VOCMeanAP(BaseMetric):
199200
Defaults to 4.
200201
drop_class_ap (bool): Whether to drop the class without ground truth
201202
when calculating the average precision for each class.
202-
classwise_result (bool): Whether to return the computed
203-
results of each class.
204-
Defaults to False.
203+
classwise (bool): Whether to return the computed results of each
204+
class. Defaults to False.
205205
**kwargs: Keyword parameters passed to :class:`BaseMetric`.
206206
207207
Examples:
@@ -241,12 +241,15 @@ def __init__(self,
241241
use_legacy_coordinate: bool = False,
242242
nproc: int = 4,
243243
drop_class_ap: bool = True,
244-
classwise_result: bool = False,
244+
classwise: bool = False,
245245
**kwargs) -> None:
246246
super().__init__(**kwargs)
247247

248248
if isinstance(iou_thrs, float):
249249
iou_thrs = [iou_thrs]
250+
assert is_list_of(iou_thrs, float), \
251+
'`iou_thrs` should be float or a list of float'
252+
250253
self.iou_thrs = iou_thrs
251254

252255
if scale_ranges is None:
@@ -272,7 +275,7 @@ def __init__(self,
272275
self.nproc = nproc
273276
self.use_legacy_coordinate = use_legacy_coordinate
274277
self.drop_class_ap = drop_class_ap
275-
self.classwise_result = classwise_result
278+
self.classwise = classwise
276279

277280
self.num_iou = len(self.iou_thrs)
278281
self.num_scale = len(self.scale_ranges)
@@ -321,20 +324,20 @@ def add(self, predictions: Sequence[Dict], groundtruths: Sequence[Dict]) -> None
321324
322325
- bboxes (numpy.ndarray): Shape (M, 4), the ground truth
323326
bounding bboxes of this image, in 'xyxy' foramrt.
324-
- labels (numpy.ndarray): Shape (M, 1), theground truth
327+
- labels (numpy.ndarray): Shape (M, 1), the ground truth
325328
labels of bounding boxes.
326329
- bboxes_ignore (numpy.ndarray): Shape (K, 4), the ground
327330
truth ignored bounding bboxes of this image,
328331
in 'xyxy' foramrt.
329332
- labels_ignore (numpy.ndarray): Shape (K, 1), the ground
330333
truth ignored labels of bounding boxes.
331334
"""
332-
for prediction, label in zip(predictions, groundtruths):
335+
for prediction, groundtruth in zip(predictions, groundtruths):
333336
assert isinstance(prediction, dict), 'The prediciton should be ' \
334337
f'a sequence of dict, but got a sequence of {type(prediction)}.' # noqa: E501
335-
assert isinstance(label, dict), 'The label should be ' \
336-
f'a sequence of dict, but got a sequence of {type(label)}.'
337-
self._results.append((prediction, label))
338+
assert isinstance(groundtruth, dict), 'The label should be ' \
339+
f'a sequence of dict, but got a sequence of {type(groundtruth)}.' # noqa: E501
340+
self._results.append((prediction, groundtruth))
338341

339342
@staticmethod
340343
def _calculate_image_tpfp(
@@ -562,8 +565,8 @@ def compute_metric(self, results: list) -> dict:
562565
- mAP, the averaged across all IoU thresholds and all class.
563566
- mAP@{IoU}, the mAP of the specified IoU threshold.
564567
- mAP@{scale_range}, the mAP of the specified scale range.
565-
- classwise_result, the evaluation results of each class.
566-
This would be returned if ``self.classwise_result`` is True.
568+
- classwise, the evaluation results of each class.
569+
This would be returned if ``self.classwise`` is True.
567570
"""
568571
predictions, groundtruths = zip(*results)
569572

@@ -605,7 +608,7 @@ def compute_metric(self, results: list) -> dict:
605608
pool.close()
606609

607610
eval_results = self._aggregate_results(results_per_class)
608-
if self.classwise_result:
611+
if self.classwise:
609612
eval_results['classwise_result'] = results_per_class
610613

611614
return eval_results

requirements/optional.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
pycocotools
12
scipy
23
shapely

0 commit comments

Comments
 (0)