1.Environment configuration
1.1 Create Conda virtual environment
!conda create -n [env_name] python=3.9
!conda activate [env_name]
1.2 Install Pytorch, etc.
!pip3 install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio==0.10.1+cu113 -f !https://download.pytorch.org/whl/cu113/torch_stable.html
!pip install -U openmim
!mim install mmengine
!mim install 'mmcv==2.0.0rc4'
1.3 Install other tool packages
!pip install opencv-python pillow matplotlib seaborn tqdm pytorch-lightning 'mmdet>=3.0.0rc1' -i https://pypi.tuna.tsinghua.edu.cn/simple
1.4 Install mmsegmentation
# 删掉原有的 mmsegmentation 文件夹(如有)
!rm -rf mmsegmentation
# 从 github 上下载最新的 mmsegmentation 源代码
!git clone https://github.com/open-mmlab/mmsegmentation.git -b dev-1.x
#切换工作目录
import os
os.chdir('mmsegmentation')
#安装mmsegmentation
!pip install -v -e .
1.5 Download pre-trained weights and data
import os
# 创建 checkpoint 文件夹,用于存放预训练模型权重文件
os.mkdir('checkpoint')
# 创建 outputs 文件夹,用于存放预测结果
os.mkdir('outputs')
# 创建 data 文件夹,用于存放图片和视频素材
os.mkdir('data')
# 从 Model Zoo 获取 PSPNet 预训练模型,下载并保存在 checkpoint 文件夹中
!wget https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r50-d8_512x1024_40k_cityscapes/pspnet_r50-d8_512x1024_40k_cityscapes_20200605_003338-2966598c.pth -P checkpoint
### 下载数据
# 伦敦街景图片
!wget https://zihao-openmmlab.obs.cn-east-3.myhuaweicloud.com/20220713-mmdetection/images/street_uk.jpeg -P data
# 上海驾车街景视频,视频来源:https://www.youtube.com/watch?v=ll8TgCZ0plk
!wget https://zihao-download.obs.cn-east-3.myhuaweicloud.com/detectron2/traffic.mp4 -P data
# 街拍视频,2022年3月30日
!wget https://zihao-openmmlab.obs.cn-east-3.myhuaweicloud.com/20220713-mmdetection/imag
1.6 Check if the installation is successful
# 检查 Pytorch
import torch, torchvision
print('Pytorch 版本', torch.__version__)
print('CUDA 是否可用',torch.cuda.is_available())
# 检查 mmcv
import mmcv
from mmcv.ops import get_compiling_cuda_version, get_compiler_version
print('MMCV版本', mmcv.__version__)
print('CUDA版本', get_compiling_cuda_version())
print('编译器版本', get_compiler_version())
# 检查 mmsegmentation
import mmseg
from mmseg.utils import register_all_modules
from mmseg.apis import inference_model, init_model
print('mmsegmentation版本', mmseg.__version__)
No error is reported, which proves that the installation is successful.
1.7 Set Matplotlib Chinese font
Install fonts
# Linux操作系统
# 如果遇到 SSL 相关报错,重新运行本代码块即可
!wget https://zihao-openmmlab.obs.cn-east-3.myhuaweicloud.com/20220716-mmclassification/dataset/SimHei.ttf -O /environment/miniconda3/lib/python3.7/site-packages/matplotlib/mpl-data/fonts/ttf/SimHei.ttf
!rm -rf /home/featurize/.cache/matplotlib
import matplotlib
import matplotlib.pyplot as plt
matplotlib.rc("font",family='SimHei') # 中文字体
test
plt.plot([1,2,3], [100,500,300])
plt.title('matplotlib中文字体测试', fontsize=25)
plt.xlabel('X轴', fontsize=15)
plt.ylabel('Y轴', fontsize=15)
plt.show()
2. Pre-training Semantic Segmentation Model Prediction - Single Image
2.1 Switch to the mmsegmentation directory
import os
os.chdir('mmsegmentation')
2.2 Load test image
from PIL import Image
# Image.open('data/street_uk.jpeg')
2.3 PSPNet
#运行demo
!python demo/image_demo.py \
data/street_uk.jpeg \
configs/pspnet/pspnet_r50-d8_4xb2-40k_cityscapes-512x1024.py \
https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r50-d8_512x1024_40k_cityscapes/pspnet_r50-d8_512x1024_40k_cityscapes_20200605_003338-2966598c.pth \
--out-file outputs/B1_uk_pspnet.jpg \
--device cuda:0 \
--opacity 0.5
# 打开输出图片
Image.open('outputs/B1_uk_pspnet.jpg')
2.3 SegFormer
#运行demo
!python demo/image_demo.py \
data/street_uk.jpeg \
configs/segformer/segformer_mit-b5_8xb1-160k_cityscapes-1024x1024.py \
https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b5_8x1_1024x1024_160k_cityscapes/segformer_mit-b5_8x1_1024x1024_160k_cityscapes_20211206_072934-87a052ec.pth \
--out-file outputs/B1_uk_segformer.jpg \
--device cuda:0 \
--opacity 0.5
# 打开输出图片
Image.open('outputs/B1_uk_segformer.jpg')
2.4 Mask2Former
#运行demo
!python demo/image_demo.py \
data/street_uk.jpeg \
configs/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024.py \
https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024_20221202_141901-28ad20f1.pth \
--out-file outputs/B1_uk_Mask2Former.jpg \
--device cuda:0 \
--opacity 0.5
# 打开输出图片
Image.open('outputs/B1_uk_Mask2Former.jpg')
Note : All the above configuration files and pre-trained models can be found on the mmsegmentation official website
Supplement: ADE20K semantic segmentation data set
The story about ADE20K: https://www.zhihu.com/question/390783647/answer/1226097849
3. Pre-training Semantic Segmentation Model Prediction-Single Image-Python API
3.1 Enter the mmsegmentation main directory
import os
os.chdir('mmsegmentation')
3.2 Image prediction-API
#导入工具包
import torch
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import mmcv
from mmseg.apis import init_model, inference_model, show_result_pyplot
from mmseg.utils import register_all_modules
register_all_modules()
#载入测试图像
from PIL import Image
# img_path = 'demo/demo.png'
img_path = 'data/street_uk.jpeg'
img_pil = Image.open(img_path)
#载入模型
# 模型 config 配置文件
# config_file = 'configs/pspnet/pspnet_r50-d8_4xb2-40k_cityscapes-512x1024.py'
config_file = 'configs/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024.py'
# 模型 checkpoint 权重文件
# checkpoint_file = 'https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r50-d8_512x1024_40k_cityscapes/pspnet_r50-d8_512x1024_40k_cityscapes_20200605_003338-2966598c.pth'
checkpoint_file = 'https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024_20221202_141901-dc2c2ddd.pth'
from mmseg.apis import init_model
model = init_model(config_file, checkpoint_file, device='cuda:0')
#运行语义分割预测
from mmseg.apis import inference_model
from mmengine.model.utils import revert_sync_batchnorm
if not torch.cuda.is_available():
model = revert_sync_batchnorm(model)
result = inference_model(model, img_path)
3.3 Analysis of Semantic Segmentation Prediction Results
result
result.key()
# 类别:0-18,共 19 个 类别
result.pred_sem_seg.data.shape
np.unique(result.pred_sem_seg.data.cpu())
result.pred_sem_seg.data.shape
result.pred_sem_seg.data
class_map = result.pred_sem_seg.data[0].detach().cpu().numpy()
plt.imshow(class_map)
plt.show()
# 置信度
result.seg_logits.data.shape
3.4 Visualizing semantic segmentation results
method one:
from mmseg.apis import show_result_pyplot
visualization = show_result_pyplot(model, img_path, result, opacity=0.8, title='MMSeg', out_file='outputs/B2.jpg')
visualization.shape
plt.imshow(mmcv.bgr2rgb(visualization))
plt.show()
Image.open('outputs/B2.jpg')
Method Two:
from mmseg.datasets import cityscapes
import numpy as np
import mmcv
# 获取类别名和调色板
classes = cityscapes.CityscapesDataset.METAINFO['classes']
palette = cityscapes.CityscapesDataset.METAINFO['palette']
opacity = 0.15 # 透明度,越大越接近原图
# 将分割图按调色板染色
# seg_map = result[0].astype('uint8')
seg_map = class_map.astype('uint8')
seg_img = Image.fromarray(seg_map).convert('P')
seg_img.putpalette(np.array(palette, dtype=np.uint8))
from matplotlib import pyplot as plt
import matplotlib.patches as mpatches
plt.figure(figsize=(14, 8))
im = plt.imshow(((np.array(seg_img.convert('RGB')))*(1-opacity) + mmcv.imread(img_path)*opacity) / 255)
# 为每一种颜色创建一个图例
patches = [mpatches.Patch(color=np.array(palette[i])/255., label=classes[i]) for i in range(18)]
plt.legend(handles=patches, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0., fontsize='large')
plt.show()
4. Pre-trained semantic segmentation model prediction-video
4.1 Enter the mmsegmentation main directory
import os
os.chdir('mmsegmentation')
4.2 Video prediction - command line (not recommended, slow)
!python demo/video_demo.py \
data/street_20220330_174028.mp4 \
configs/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024.py \
https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024_20221202_141901-28ad20f1.pth \
--device cuda:0 \
--output-file outputs/B3_video.mp4 \
--opacity 0.5
4.3 Video prediction-Python API (recommended, fast)
#导入工具包
import numpy as np
import time
import shutil
import torch
from PIL import Image
import cv2
import mmcv
import mmengine
from mmseg.apis import inference_model
from mmseg.utils import register_all_modules
register_all_modules()
from mmseg.datasets import CityscapesDataset
#载入模型
# 模型 config 配置文件
config_file = 'configs/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024.py'
# 模型 checkpoint 权重文件
checkpoint_file = 'https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024/mask2former_swin-l-in22k-384x384-pre_8xb2-90k_cityscapes-512x1024_20221202_141901-28ad20f1.pth'
from mmseg.apis import init_model
model = init_model(config_file, checkpoint_file, device='cuda:0')
from mmengine.model.utils import revert_sync_batchnorm
if not torch.cuda.is_available():
model = revert_sync_batchnorm(model)
#输入视频路径
input_video = 'data/street_20220330_174028.mp4'
#创建临时文件夹,存放每帧结果
temp_out_dir = time.strftime('%Y%m%d%H%M%S')
os.mkdir(temp_out_dir)
print('创建临时文件夹 {} 用于存放每帧预测结果'.format(temp_out_dir))
#视频单帧预测
# 获取 Cityscapes 街景数据集 类别名和调色板
from mmseg.datasets import cityscapes
classes = cityscapes.CityscapesDataset.METAINFO['classes']
palette = cityscapes.CityscapesDataset.METAINFO['palette']
def pridict_single_frame(img, opacity=0.2):
result = inference_model(model, img)
# 将分割图按调色板染色
seg_map = np.array(result.pred_sem_seg.data[0].detach().cpu().numpy()).astype('uint8')
seg_img = Image.fromarray(seg_map).convert('P')
seg_img.putpalette(np.array(palette, dtype=np.uint8))
show_img = (np.array(seg_img.convert('RGB')))*(1-opacity) + img*opacity
return show_img
#视频逐帧预测
# 读入待预测视频
imgs = mmcv.VideoReader(input_video)
prog_bar = mmengine.ProgressBar(len(imgs))
# 对视频逐帧处理
for frame_id, img in enumerate(imgs):
## 处理单帧画面
show_img = pridict_single_frame(img, opacity=0.15)
temp_path = f'{
temp_out_dir}/{
frame_id:06d}.jpg' # 保存语义分割预测结果图像至临时文件夹
cv2.imwrite(temp_path, show_img)
prog_bar.update() # 更新进度条
# 把每一帧串成视频文件
mmcv.frames2video(temp_out_dir, 'outputs/B3_video.mp4', fps=imgs.fps, fourcc='mp4v')
shutil.rmtree(temp_out_dir) # 删除存放每帧画面的临时文件夹
print('删除临时文件夹', temp_out_dir)
5. Custom MMSegmentation dataset
The basic steps:
- Download the curated dataset;
- Visually explore datasets;
- Prepare the config configuration file;
- MMSef trains semantic segmentation model;
- Visual training log;
- Use the trained model to predict;
- Test set performance evaluation;