diff --git a/tools/analysis_tools/coco_error_analysis.py b/tools/analysis_tools/coco_error_analysis.py
index fba96cafd2e818afacc43b120793dbb1dd228705..722efe6d31f07ba92ecb14836ccd834d5a318bb0 100644
--- a/tools/analysis_tools/coco_error_analysis.py
+++ b/tools/analysis_tools/coco_error_analysis.py
@@ -12,17 +12,17 @@ from pycocotools.cocoeval import COCOeval
def makeplot(rs, ps, outDir, class_name, iou_type):
cs = np.vstack([
np.ones((2, 3)),
- np.array([.31, .51, .74]),
- np.array([.75, .31, .30]),
- np.array([.36, .90, .38]),
- np.array([.50, .39, .64]),
- np.array([1, .6, 0])
+ np.array([0.31, 0.51, 0.74]),
+ np.array([0.75, 0.31, 0.30]),
+ np.array([0.36, 0.90, 0.38]),
+ np.array([0.50, 0.39, 0.64]),
+ np.array([1, 0.6, 0]),
])
areaNames = ['allarea', 'small', 'medium', 'large']
types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
for i in range(len(areaNames)):
area_ps = ps[..., i, 0]
- figure_tile = iou_type + '-' + class_name + '-' + areaNames[i]
+ figure_title = iou_type + '-' + class_name + '-' + areaNames[i]
aps = [ps_.mean() for ps_ in area_ps]
ps_curve = [
ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps
@@ -37,19 +37,145 @@ def makeplot(rs, ps, outDir, class_name, iou_type):
ps_curve[k],
ps_curve[k + 1],
color=cs[k],
- label=str(f'[{aps[k]:.3f}]' + types[k]))
+ label=str(f'[{aps[k]:.3f}]' + types[k]),
+ )
plt.xlabel('recall')
plt.ylabel('precision')
- plt.xlim(0, 1.)
- plt.ylim(0, 1.)
- plt.title(figure_tile)
+ plt.xlim(0, 1.0)
+ plt.ylim(0, 1.0)
+ plt.title(figure_title)
plt.legend()
# plt.show()
- fig.savefig(outDir + f'/{figure_tile}.png')
+ fig.savefig(outDir + f'/{figure_title}.png')
plt.close(fig)
-def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
+def autolabel(ax, rects):
+ """Attach a text label above each bar in *rects*, displaying its height."""
+ for rect in rects:
+ height = rect.get_height()
+ if height > 0 and height <= 1: # for percent values
+ text_label = '{:2.0f}'.format(height * 100)
+ else:
+ text_label = '{:2.0f}'.format(height)
+ ax.annotate(
+ text_label,
+ xy=(rect.get_x() + rect.get_width() / 2, height),
+ xytext=(0, 3), # 3 points vertical offset
+ textcoords='offset points',
+ ha='center',
+ va='bottom',
+ fontsize='x-small',
+ )
+
+
+def makebarplot(rs, ps, outDir, class_name, iou_type):
+ areaNames = ['allarea', 'small', 'medium', 'large']
+ types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
+ fig, ax = plt.subplots()
+ x = np.arange(len(areaNames)) # the areaNames locations
+ width = 0.60 # the width of the bars
+ rects_list = []
+ figure_title = iou_type + '-' + class_name + '-' + 'ap bar plot'
+ for i in range(len(types) - 1):
+ type_ps = ps[i, ..., 0]
+ aps = [ps_.mean() for ps_ in type_ps.T]
+ rects_list.append(
+ ax.bar(
+ x - width / 2 + (i + 1) * width / len(types),
+ aps,
+ width / len(types),
+ label=types[i],
+ ))
+
+ # Add some text for labels, title and custom x-axis tick labels, etc.
+ ax.set_ylabel('Mean Average Precision (mAP)')
+ ax.set_title(figure_title)
+ ax.set_xticks(x)
+ ax.set_xticklabels(areaNames)
+ ax.legend()
+
+ # Add score texts over bars
+ for rects in rects_list:
+ autolabel(ax, rects)
+
+ # Save plot
+ fig.savefig(outDir + f'/{figure_title}.png')
+ plt.close(fig)
+
+
+def get_gt_area_group_numbers(cocoEval):
+ areaRng = cocoEval.params.areaRng
+ areaRngStr = [str(aRng) for aRng in areaRng]
+ areaRngLbl = cocoEval.params.areaRngLbl
+ areaRngStr2areaRngLbl = dict(zip(areaRngStr, areaRngLbl))
+ areaRngLbl2Number = dict.fromkeys(areaRngLbl, 0)
+ for evalImg in cocoEval.evalImgs:
+ if evalImg:
+ for gtIgnore in evalImg['gtIgnore']:
+ if not gtIgnore:
+ aRngLbl = areaRngStr2areaRngLbl[str(evalImg['aRng'])]
+ areaRngLbl2Number[aRngLbl] += 1
+ return areaRngLbl2Number
+
+
+def make_gt_area_group_numbers_plot(cocoEval, outDir, verbose=True):
+ areaRngLbl2Number = get_gt_area_group_numbers(cocoEval)
+ areaRngLbl = areaRngLbl2Number.keys()
+ if verbose:
+ print('number of annotations per area group:', areaRngLbl2Number)
+
+ # Init figure
+ fig, ax = plt.subplots()
+ x = np.arange(len(areaRngLbl)) # the areaNames locations
+ width = 0.60 # the width of the bars
+ figure_title = 'number of annotations per area group'
+
+ rects = ax.bar(x, areaRngLbl2Number.values(), width)
+
+ # Add some text for labels, title and custom x-axis tick labels, etc.
+ ax.set_ylabel('Number of annotations')
+ ax.set_title(figure_title)
+ ax.set_xticks(x)
+ ax.set_xticklabels(areaRngLbl)
+
+ # Add score texts over bars
+ autolabel(ax, rects)
+
+ # Save plot
+ fig.tight_layout()
+ fig.savefig(outDir + f'/{figure_title}.png')
+ plt.close(fig)
+
+
+def make_gt_area_histogram_plot(cocoEval, outDir):
+ n_bins = 100
+ areas = [ann['area'] for ann in cocoEval.cocoGt.anns.values()]
+
+ # init figure
+ figure_title = 'gt annotation areas histogram plot'
+ fig, ax = plt.subplots()
+
+ # Set the number of bins
+ ax.hist(np.sqrt(areas), bins=n_bins)
+
+ # Add some text for labels, title and custom x-axis tick labels, etc.
+ ax.set_xlabel('Squareroot Area')
+ ax.set_ylabel('Number of annotations')
+ ax.set_title(figure_title)
+
+ # Save plot
+ fig.tight_layout()
+ fig.savefig(outDir + f'/{figure_title}.png')
+ plt.close(fig)
+
+
+def analyze_individual_category(k,
+ cocoDt,
+ cocoGt,
+ catId,
+ iou_type,
+ areas=None):
nm = cocoGt.loadCats(catId)[0]
print(f'--------------analyzing {k + 1}-{nm["name"]}---------------')
ps_ = {}
@@ -67,16 +193,18 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
gt = copy.deepcopy(cocoGt)
child_catIds = gt.getCatIds(supNms=[nm['supercategory']])
for idx, ann in enumerate(gt.dataset['annotations']):
- if (ann['category_id'] in child_catIds
- and ann['category_id'] != catId):
+ if ann['category_id'] in child_catIds and ann['category_id'] != catId:
gt.dataset['annotations'][idx]['ignore'] = 1
gt.dataset['annotations'][idx]['iscrowd'] = 1
gt.dataset['annotations'][idx]['category_id'] = catId
cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
cocoEval.params.imgIds = imgIds
cocoEval.params.maxDets = [100]
- cocoEval.params.iouThrs = [.1]
+ cocoEval.params.iouThrs = [0.1]
cocoEval.params.useCats = 1
+ if areas:
+ cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+ [areas[0], areas[1]], [areas[1], areas[2]]]
cocoEval.evaluate()
cocoEval.accumulate()
ps_supercategory = cocoEval.eval['precision'][0, :, k, :, :]
@@ -91,8 +219,11 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
cocoEval.params.imgIds = imgIds
cocoEval.params.maxDets = [100]
- cocoEval.params.iouThrs = [.1]
+ cocoEval.params.iouThrs = [0.1]
cocoEval.params.useCats = 1
+ if areas:
+ cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+ [areas[0], areas[1]], [areas[1], areas[2]]]
cocoEval.evaluate()
cocoEval.accumulate()
ps_allcategory = cocoEval.eval['precision'][0, :, k, :, :]
@@ -100,9 +231,17 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
return k, ps_
-def analyze_results(res_file, ann_file, res_types, out_dir):
+def analyze_results(res_file,
+ ann_file,
+ res_types,
+ out_dir,
+ extraplots=None,
+ areas=None):
for res_type in res_types:
assert res_type in ['bbox', 'segm']
+ if areas:
+ assert len(areas) == 3, '3 integers should be specified as areas, \
+ representing 3 area regions'
directory = os.path.dirname(out_dir + '/')
if not os.path.exists(directory):
@@ -122,8 +261,12 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
cocoEval = COCOeval(
copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type)
cocoEval.params.imgIds = imgIds
- cocoEval.params.iouThrs = [.75, .5, .1]
+ cocoEval.params.iouThrs = [0.75, 0.5, 0.1]
cocoEval.params.maxDets = [100]
+ if areas:
+ cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+ [areas[0], areas[1]],
+ [areas[1], areas[2]]]
cocoEval.evaluate()
cocoEval.accumulate()
ps = cocoEval.eval['precision']
@@ -131,7 +274,7 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
catIds = cocoGt.getCatIds()
recThrs = cocoEval.params.recThrs
with Pool(processes=48) as pool:
- args = [(k, cocoDt, cocoGt, catId, iou_type)
+ args = [(k, cocoDt, cocoGt, catId, iou_type, areas)
for k, catId in enumerate(catIds)]
analyze_results = pool.starmap(analyze_individual_category, args)
for k, catId in enumerate(catIds):
@@ -147,10 +290,18 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
ps[4, :, k, :, :] = ps_allcategory
# fill in background and false negative errors and plot
ps[ps == -1] = 0
- ps[5, :, k, :, :] = (ps[4, :, k, :, :] > 0)
+ ps[5, :, k, :, :] = ps[4, :, k, :, :] > 0
ps[6, :, k, :, :] = 1.0
makeplot(recThrs, ps[:, :, k], res_out_dir, nm['name'], iou_type)
+ if extraplots:
+ makebarplot(recThrs, ps[:, :, k], res_out_dir, nm['name'],
+ iou_type)
makeplot(recThrs, ps, res_out_dir, 'allclass', iou_type)
+ if extraplots:
+ makebarplot(recThrs, ps, res_out_dir, 'allclass', iou_type)
+ make_gt_area_group_numbers_plot(
+ cocoEval=cocoEval, outDir=res_out_dir, verbose=True)
+ make_gt_area_histogram_plot(cocoEval=cocoEval, outDir=res_out_dir)
def main():
@@ -163,8 +314,24 @@ def main():
help='annotation file path')
parser.add_argument(
'--types', type=str, nargs='+', default=['bbox'], help='result types')
+ parser.add_argument(
+ '--extraplots',
+ action='store_true',
+ help='export extra bar/stat plots')
+ parser.add_argument(
+ '--areas',
+ type=int,
+ nargs='+',
+ default=[1024, 9216, 10000000000],
+ help='area regions')
args = parser.parse_args()
- analyze_results(args.result, args.ann, args.types, out_dir=args.out_dir)
+ analyze_results(
+ args.result,
+ args.ann,
+ args.types,
+ out_dir=args.out_dir,
+ extraplots=args.extraplots,
+ areas=args.areas)
if __name__ == '__main__':