diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 38064c446aac2f0836cab04c941640aaf7b5d4d3..24b7e53dba94cd31e18922c96ad3d2a24835131e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -52,7 +52,6 @@ jobs:
         run: |
           pip install mmcv-full==latest+torch${{matrix.torch}}+cu101 -f https://openmmlab.oss-accelerate.aliyuncs.com/mmcv/dist/index.html
           pip install -r requirements.txt
-          pip install "git+https://github.com/open-mmlab/cocoapi.git#subdirectory=pycocotools"
       - name: Lint with flake8
         run: flake8 .
       - name: Lint with isort
diff --git a/configs/lvis/README.md b/configs/lvis/README.md
index a705955562dd89d524982da2e418699ef76ef2d0..3539e5f9169065031392fc6eaf53a1f11935ac13 100644
--- a/configs/lvis/README.md
+++ b/configs/lvis/README.md
@@ -16,6 +16,10 @@
     ```
     pip install "git+https://github.com/open-mmlab/cocoapi.git#subdirectory=lvis"
     ```
+    or 
+    ```
+    pip install -r requirements/optional.txt
+    ```
 * All experiments use oversample strategy [here](../../docs/tutorials/new_dataset.md#class-balanced-dataset) with oversample threshold `1e-3`.
 * The size of LVIS v0.5 is half of COCO, so schedule `2x` in LVIS is roughly the same iterations as `1x` in COCO.
 
diff --git a/docs/install.md b/docs/install.md
index 92f21728a83efa1a3cb10568e3fe3e3701d7fc12..b96cdaa5b6f969735d7256691f5ec51320a8345a 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -74,7 +74,6 @@ for better compatibility with our repo.)
 
 ```shell
 pip install -r requirements/build.txt
-pip install "git+https://github.com/open-mmlab/cocoapi.git#subdirectory=pycocotools"
 pip install -v -e .  # or "python setup.py develop"
 ```
 
@@ -152,7 +151,6 @@ pip install mmcv-full
 git clone https://github.com/open-mmlab/mmdetection.git
 cd mmdetection
 pip install -r requirements/build.txt
-pip install "git+https://github.com/open-mmlab/cocoapi.git#subdirectory=pycocotools"
 pip install -v -e .
 ```
 
diff --git a/requirements/optional.txt b/requirements/optional.txt
index b34b112abb2b40b0b19e0963e7551bde45a68c0f..3c406c97cca304a6baeae20e38b13450a138b145 100644
--- a/requirements/optional.txt
+++ b/requirements/optional.txt
@@ -1,3 +1,4 @@
 albumentations>=0.3.2
 cityscapesscripts
 imagecorruptions
+lvis@git+https://github.com/open-mmlab/cocoapi.git#subdirectory=lvis
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index 800d1238e6c96e354207e9ce43181cb8ea6eff79..786c9065ca327a104b5ab71bdc79e03ae51d7145 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -6,3 +6,4 @@ six
 terminaltables
 torch>=1.3
 torchvision
+pycocotools@git+https://github.com/open-mmlab/cocoapi.git#subdirectory=pycocotools
diff --git a/setup.py b/setup.py
index f67b36beee23c34ffa3d2d321353974082f41922..f26ad3b3bd884b82140d3fbdbf17a9c461e2199d 100755
--- a/setup.py
+++ b/setup.py
@@ -142,6 +142,8 @@ def parse_requirements(fname='requirements.txt', with_version=True):
             info = {'line': line}
             if line.startswith('-e '):
                 info['package'] = line.split('#egg=')[1]
+            elif '@git+' in line:
+                info['package'] = line
             else:
                 # Remove versioning from the package
                 pat = '(' + '|'.join(['>=', '==', '>']) + ')'