Add fsaf into regression benchmark (#2581)

* Add fsaf into regression benchmark

* Cover more cases in unittest & regression benchmark

* Minor fix

* rm nas_fpn from test_pipeline

* avoid duplicated cfgs

.dev_scripts/benchmark_filter.py

@@ -29,12 +29,14 @@ def parse_args():
 basic_arch_root = [
     'cascade_rcnn', 'double_heads', 'fcos', 'foveabox', 'free_anchor',
     'grid_rcnn', 'guided_anchoring', 'htc', 'libra_rcnn', 'atss', 'mask_rcnn',
-    'ms_rcnn', 'nas_fpn', 'reppoints', 'retinanet', 'ssd', 'gn', 'ghm'
+    'ms_rcnn', 'nas_fpn', 'reppoints', 'retinanet', 'ssd', 'gn', 'ghm', 'fsaf'
 ]
 
 datasets_root = ['wider_face', 'pascal_voc', 'cityscapes', 'mask_rcnn']
 
-data_pipeline_root = ['albu_example', 'instaboost']
+data_pipeline_root = [
+    'albu_example', 'instaboost', 'ssd', 'mask_rcnn', 'nas_fpn'
+]
 
 nn_module_root = [
     'carafe', 'dcn', 'empirical_attention', 'gcnet', 'gn+ws', 'hrnet', 'pafpn',
@@ -64,6 +66,7 @@ benchmark_pool = [
     'configs/reppoints/reppoints_moment_r50_fpn_gn-neck+head_1x_coco.py',
     'configs/guided_anchoring/ga_faster_r50_caffe_fpn_1x_coco.py',
     'configs/free_anchor/retinanet_free_anchor_r50_fpn_1x_coco.py',
+    'configs/fsaf/fsaf_r50_fpn_1x_coco.py',
     'configs/scratch/mask_rcnn_r50_fpn_gn-all_scratch_6x_coco.py',
     'configs/pafpn/faster_rcnn_r50_pafpn_1x_coco.py',
     'configs/fp16/retinanet_r50_fpn_fp16_1x_coco.py',
@@ -80,7 +83,7 @@ benchmark_pool = [
     'configs/pascal_voc/faster_rcnn_r50_fpn_1x_voc0712.py',
     'configs/pascal_voc/ssd300_voc0712.py',
     'configs/nas_fpn/retinanet_r50_nasfpn_crop640_50e_coco.py',
-    'configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_1x_coco.py',
+    'configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_mstrain-poly_1x_coco.py',
     'configs/gn+ws/mask_rcnn_r50_fpn_gn_ws-all_2x_coco.py'
 ]
 
@@ -105,7 +108,8 @@ def main():
         configs = os.scandir(cfg_dir)
         for cfg in configs:
             config_path = osp.join(cfg_dir, cfg.name)
-            if config_path in benchmark_pool:
+            if (config_path in benchmark_pool
+                    and config_path not in benchmark_configs):
                 benchmark_configs.append(config_path)
 
     print(f'Totally found {len(benchmark_configs)} configs to benchmark')

configs/carafe/README.md

@@ -42,7 +42,7 @@ Git clone mmdetection.
 git clone https://github.com/open-mmlab/mmdetection.git
 cd mmdetection
 ```
-Setup CARAFE in our project.
+Setup CARAFE in your own project.
 ```shell
 cp -r ./mmdet/ops/carafe $Your_Project_Path$
 cd $Your_Project_Path$/carafe

configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_detectron2-poly_1x_coco.py

-_base_ = './mask_rcnn_r50_fpn_1x_coco.py'
-model = dict(
-    pretrained='open-mmlab://resnet50_caffe_caffe',
-    backbone=dict(norm_cfg=dict(requires_grad=False), style='caffe'))
-# use caffe img_norm
-img_norm_cfg = dict(
-    mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False)
-train_pipeline = [
-    dict(type='LoadImageFromFile'),
-    dict(
-        type='LoadAnnotations',
-        with_bbox=True,
-        with_mask=True,
-        poly2mask=False),
-    dict(
-        type='Resize',
-        img_scale=[(1333, 640), (1333, 672), (1333, 704), (1333, 736),
-                   (1333, 768), (1333, 800)],
-        multiscale_mode='value',
-        keep_ratio=True),
-    dict(type='RandomFlip', flip_ratio=0.5),
-    dict(type='Normalize', **img_norm_cfg),
-    dict(type='Pad', size_divisor=32),
-    dict(type='DefaultFormatBundle'),
-    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']),
-]
-test_pipeline = [
-    dict(type='LoadImageFromFile'),
-    dict(
-        type='MultiScaleFlipAug',
-        img_scale=(1333, 800),
-        flip=False,
-        transforms=[
-            dict(type='Resize', keep_ratio=True),
-            dict(type='RandomFlip'),
-            dict(type='Normalize', **img_norm_cfg),
-            dict(type='Pad', size_divisor=32),
-            dict(type='ImageToTensor', keys=['img']),
-            dict(type='Collect', keys=['img']),
-        ])
-]
-data = dict(
-    train=dict(pipeline=train_pipeline),
-    val=dict(pipeline=test_pipeline),
-    test=dict(pipeline=test_pipeline))

configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_detectron2_1x_coco.py

-_base_ = './mask_rcnn_r50_fpn_1x_coco.py'
-model = dict(
-    pretrained='open-mmlab://resnet50_caffe_bgr',
-    backbone=dict(norm_cfg=dict(requires_grad=False), style='caffe'))
-# use caffe img_norm
-img_norm_cfg = dict(
-    mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False)
-train_pipeline = [
-    dict(type='LoadImageFromFile'),
-    dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
-    dict(
-        type='Resize',
-        img_scale=[(1333, 640), (1333, 672), (1333, 704), (1333, 736),
-                   (1333, 768), (1333, 800)],
-        multiscale_mode='value',
-        keep_ratio=True),
-    dict(type='RandomFlip', flip_ratio=0.5),
-    dict(type='Normalize', **img_norm_cfg),
-    dict(type='Pad', size_divisor=32),
-    dict(type='DefaultFormatBundle'),
-    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks']),
-]
-test_pipeline = [
-    dict(type='LoadImageFromFile'),
-    dict(
-        type='MultiScaleFlipAug',
-        img_scale=(1333, 800),
-        flip=False,
-        transforms=[
-            dict(type='Resize', keep_ratio=True),
-            dict(type='RandomFlip'),
-            dict(type='Normalize', **img_norm_cfg),
-            dict(type='Pad', size_divisor=32),
-            dict(type='ImageToTensor', keys=['img']),
-            dict(type='Collect', keys=['img']),
-        ])
-]
-data = dict(
-    train=dict(pipeline=train_pipeline),
-    val=dict(pipeline=test_pipeline),
-    test=dict(pipeline=test_pipeline))

tests/test_config.py

@@ -97,7 +97,8 @@ def test_config_data_pipeline():
         'pascal_voc/ssd512_voc0712.py',
         # 'albu_example/mask_rcnn_r50_fpn_1x.py',
         'foveabox/fovea_align_r50_fpn_gn-head_mstrain_640-800_4x4_2x_coco.py',
-        'mask_rcnn/mask_rcnn_r50_fpn_poly_1x_coco.py',
+        'mask_rcnn/mask_rcnn_r50_caffe_fpn_mstrain-poly_1x_coco.py',
+        'mask_rcnn/mask_rcnn_r50_caffe_fpn_mstrain_1x_coco.py',
         'fp16/mask_rcnn_r50_fpn_fp16_1x_coco.py',
     ]