[feature] improve coco_error_analysis.py (#4712)

* add area name grouped map bar plots

* update function name, add comments, fix typo

* make all changes optional, add adjustable area region support, add area stat plots

* clean unnecessary comma

tools/analysis_tools/coco_error_analysis.py

@@ -12,17 +12,17 @@ from pycocotools.cocoeval import COCOeval
 def makeplot(rs, ps, outDir, class_name, iou_type):
     cs = np.vstack([
         np.ones((2, 3)),
-        np.array([.31, .51, .74]),
-        np.array([.75, .31, .30]),
-        np.array([.36, .90, .38]),
-        np.array([.50, .39, .64]),
-        np.array([1, .6, 0])
+        np.array([0.31, 0.51, 0.74]),
+        np.array([0.75, 0.31, 0.30]),
+        np.array([0.36, 0.90, 0.38]),
+        np.array([0.50, 0.39, 0.64]),
+        np.array([1, 0.6, 0]),
     ])
     areaNames = ['allarea', 'small', 'medium', 'large']
     types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
     for i in range(len(areaNames)):
         area_ps = ps[..., i, 0]
-        figure_tile = iou_type + '-' + class_name + '-' + areaNames[i]
+        figure_title = iou_type + '-' + class_name + '-' + areaNames[i]
         aps = [ps_.mean() for ps_ in area_ps]
         ps_curve = [
             ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps
@@ -37,19 +37,145 @@ def makeplot(rs, ps, outDir, class_name, iou_type):
                 ps_curve[k],
                 ps_curve[k + 1],
                 color=cs[k],
-                label=str(f'[{aps[k]:.3f}]' + types[k]))
+                label=str(f'[{aps[k]:.3f}]' + types[k]),
+            )
         plt.xlabel('recall')
         plt.ylabel('precision')
-        plt.xlim(0, 1.)
-        plt.ylim(0, 1.)
-        plt.title(figure_tile)
+        plt.xlim(0, 1.0)
+        plt.ylim(0, 1.0)
+        plt.title(figure_title)
         plt.legend()
         # plt.show()
-        fig.savefig(outDir + f'/{figure_tile}.png')
+        fig.savefig(outDir + f'/{figure_title}.png')
         plt.close(fig)
 
 
-def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
+def autolabel(ax, rects):
+    """Attach a text label above each bar in *rects*, displaying its height."""
+    for rect in rects:
+        height = rect.get_height()
+        if height > 0 and height <= 1:  # for percent values
+            text_label = '{:2.0f}'.format(height * 100)
+        else:
+            text_label = '{:2.0f}'.format(height)
+        ax.annotate(
+            text_label,
+            xy=(rect.get_x() + rect.get_width() / 2, height),
+            xytext=(0, 3),  # 3 points vertical offset
+            textcoords='offset points',
+            ha='center',
+            va='bottom',
+            fontsize='x-small',
+        )
+
+
+def makebarplot(rs, ps, outDir, class_name, iou_type):
+    areaNames = ['allarea', 'small', 'medium', 'large']
+    types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
+    fig, ax = plt.subplots()
+    x = np.arange(len(areaNames))  # the areaNames locations
+    width = 0.60  # the width of the bars
+    rects_list = []
+    figure_title = iou_type + '-' + class_name + '-' + 'ap bar plot'
+    for i in range(len(types) - 1):
+        type_ps = ps[i, ..., 0]
+        aps = [ps_.mean() for ps_ in type_ps.T]
+        rects_list.append(
+            ax.bar(
+                x - width / 2 + (i + 1) * width / len(types),
+                aps,
+                width / len(types),
+                label=types[i],
+            ))
+
+    # Add some text for labels, title and custom x-axis tick labels, etc.
+    ax.set_ylabel('Mean Average Precision (mAP)')
+    ax.set_title(figure_title)
+    ax.set_xticks(x)
+    ax.set_xticklabels(areaNames)
+    ax.legend()
+
+    # Add score texts over bars
+    for rects in rects_list:
+        autolabel(ax, rects)
+
+    # Save plot
+    fig.savefig(outDir + f'/{figure_title}.png')
+    plt.close(fig)
+
+
+def get_gt_area_group_numbers(cocoEval):
+    areaRng = cocoEval.params.areaRng
+    areaRngStr = [str(aRng) for aRng in areaRng]
+    areaRngLbl = cocoEval.params.areaRngLbl
+    areaRngStr2areaRngLbl = dict(zip(areaRngStr, areaRngLbl))
+    areaRngLbl2Number = dict.fromkeys(areaRngLbl, 0)
+    for evalImg in cocoEval.evalImgs:
+        if evalImg:
+            for gtIgnore in evalImg['gtIgnore']:
+                if not gtIgnore:
+                    aRngLbl = areaRngStr2areaRngLbl[str(evalImg['aRng'])]
+                    areaRngLbl2Number[aRngLbl] += 1
+    return areaRngLbl2Number
+
+
+def make_gt_area_group_numbers_plot(cocoEval, outDir, verbose=True):
+    areaRngLbl2Number = get_gt_area_group_numbers(cocoEval)
+    areaRngLbl = areaRngLbl2Number.keys()
+    if verbose:
+        print('number of annotations per area group:', areaRngLbl2Number)
+
+    # Init figure
+    fig, ax = plt.subplots()
+    x = np.arange(len(areaRngLbl))  # the areaNames locations
+    width = 0.60  # the width of the bars
+    figure_title = 'number of annotations per area group'
+
+    rects = ax.bar(x, areaRngLbl2Number.values(), width)
+
+    # Add some text for labels, title and custom x-axis tick labels, etc.
+    ax.set_ylabel('Number of annotations')
+    ax.set_title(figure_title)
+    ax.set_xticks(x)
+    ax.set_xticklabels(areaRngLbl)
+
+    # Add score texts over bars
+    autolabel(ax, rects)
+
+    # Save plot
+    fig.tight_layout()
+    fig.savefig(outDir + f'/{figure_title}.png')
+    plt.close(fig)
+
+
+def make_gt_area_histogram_plot(cocoEval, outDir):
+    n_bins = 100
+    areas = [ann['area'] for ann in cocoEval.cocoGt.anns.values()]
+
+    # init figure
+    figure_title = 'gt annotation areas histogram plot'
+    fig, ax = plt.subplots()
+
+    # Set the number of bins
+    ax.hist(np.sqrt(areas), bins=n_bins)
+
+    # Add some text for labels, title and custom x-axis tick labels, etc.
+    ax.set_xlabel('Squareroot Area')
+    ax.set_ylabel('Number of annotations')
+    ax.set_title(figure_title)
+
+    # Save plot
+    fig.tight_layout()
+    fig.savefig(outDir + f'/{figure_title}.png')
+    plt.close(fig)
+
+
+def analyze_individual_category(k,
+                                cocoDt,
+                                cocoGt,
+                                catId,
+                                iou_type,
+                                areas=None):
     nm = cocoGt.loadCats(catId)[0]
     print(f'--------------analyzing {k + 1}-{nm["name"]}---------------')
     ps_ = {}
@@ -67,16 +193,18 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
     gt = copy.deepcopy(cocoGt)
     child_catIds = gt.getCatIds(supNms=[nm['supercategory']])
     for idx, ann in enumerate(gt.dataset['annotations']):
-        if (ann['category_id'] in child_catIds
-                and ann['category_id'] != catId):
+        if ann['category_id'] in child_catIds and ann['category_id'] != catId:
             gt.dataset['annotations'][idx]['ignore'] = 1
             gt.dataset['annotations'][idx]['iscrowd'] = 1
             gt.dataset['annotations'][idx]['category_id'] = catId
     cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
     cocoEval.params.imgIds = imgIds
     cocoEval.params.maxDets = [100]
-    cocoEval.params.iouThrs = [.1]
+    cocoEval.params.iouThrs = [0.1]
     cocoEval.params.useCats = 1
+    if areas:
+        cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+                                   [areas[0], areas[1]], [areas[1], areas[2]]]
     cocoEval.evaluate()
     cocoEval.accumulate()
     ps_supercategory = cocoEval.eval['precision'][0, :, k, :, :]
@@ -91,8 +219,11 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
     cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
     cocoEval.params.imgIds = imgIds
     cocoEval.params.maxDets = [100]
-    cocoEval.params.iouThrs = [.1]
+    cocoEval.params.iouThrs = [0.1]
     cocoEval.params.useCats = 1
+    if areas:
+        cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+                                   [areas[0], areas[1]], [areas[1], areas[2]]]
     cocoEval.evaluate()
     cocoEval.accumulate()
     ps_allcategory = cocoEval.eval['precision'][0, :, k, :, :]
@@ -100,9 +231,17 @@ def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
     return k, ps_
 
 
-def analyze_results(res_file, ann_file, res_types, out_dir):
+def analyze_results(res_file,
+                    ann_file,
+                    res_types,
+                    out_dir,
+                    extraplots=None,
+                    areas=None):
     for res_type in res_types:
         assert res_type in ['bbox', 'segm']
+    if areas:
+        assert len(areas) == 3, '3 integers should be specified as areas, \
+            representing 3 area regions'
 
     directory = os.path.dirname(out_dir + '/')
     if not os.path.exists(directory):
@@ -122,8 +261,12 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
         cocoEval = COCOeval(
             copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type)
         cocoEval.params.imgIds = imgIds
-        cocoEval.params.iouThrs = [.75, .5, .1]
+        cocoEval.params.iouThrs = [0.75, 0.5, 0.1]
         cocoEval.params.maxDets = [100]
+        if areas:
+            cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]],
+                                       [areas[0], areas[1]],
+                                       [areas[1], areas[2]]]
         cocoEval.evaluate()
         cocoEval.accumulate()
         ps = cocoEval.eval['precision']
@@ -131,7 +274,7 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
         catIds = cocoGt.getCatIds()
         recThrs = cocoEval.params.recThrs
         with Pool(processes=48) as pool:
-            args = [(k, cocoDt, cocoGt, catId, iou_type)
+            args = [(k, cocoDt, cocoGt, catId, iou_type, areas)
                     for k, catId in enumerate(catIds)]
             analyze_results = pool.starmap(analyze_individual_category, args)
         for k, catId in enumerate(catIds):
@@ -147,10 +290,18 @@ def analyze_results(res_file, ann_file, res_types, out_dir):
             ps[4, :, k, :, :] = ps_allcategory
             # fill in background and false negative errors and plot
             ps[ps == -1] = 0
-            ps[5, :, k, :, :] = (ps[4, :, k, :, :] > 0)
+            ps[5, :, k, :, :] = ps[4, :, k, :, :] > 0
             ps[6, :, k, :, :] = 1.0
             makeplot(recThrs, ps[:, :, k], res_out_dir, nm['name'], iou_type)
+            if extraplots:
+                makebarplot(recThrs, ps[:, :, k], res_out_dir, nm['name'],
+                            iou_type)
         makeplot(recThrs, ps, res_out_dir, 'allclass', iou_type)
+        if extraplots:
+            makebarplot(recThrs, ps, res_out_dir, 'allclass', iou_type)
+            make_gt_area_group_numbers_plot(
+                cocoEval=cocoEval, outDir=res_out_dir, verbose=True)
+            make_gt_area_histogram_plot(cocoEval=cocoEval, outDir=res_out_dir)
 
 
 def main():
@@ -163,8 +314,24 @@ def main():
         help='annotation file path')
     parser.add_argument(
         '--types', type=str, nargs='+', default=['bbox'], help='result types')
+    parser.add_argument(
+        '--extraplots',
+        action='store_true',
+        help='export extra bar/stat plots')
+    parser.add_argument(
+        '--areas',
+        type=int,
+        nargs='+',
+        default=[1024, 9216, 10000000000],
+        help='area regions')
     args = parser.parse_args()
-    analyze_results(args.result, args.ann, args.types, out_dir=args.out_dir)
+    analyze_results(
+        args.result,
+        args.ann,
+        args.types,
+        out_dir=args.out_dir,
+        extraplots=args.extraplots,
+        areas=args.areas)
 
 
 if __name__ == '__main__':