1. Install and configure MMagic
1.1 Install Pytorch
# 安装Pytorch
!pip3 install install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio==0.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
1.2 Install MMCV and MMEngine environment
!pip3 install openmim
!mim install 'mmcv>=2.0.0'
!mim install 'mmengine'
1.3 Install MMagic
method one:
!mim install 'mmagic'
Method 2: Source code installation
!rm -rf mmagic # 删除原有的 mmagic 文件夹(如有)
!git clone https://github.com/open-mmlab/mmagic.git # 下载 mmagic 源代码
import os
os.chdir('mmagic')
!pip3 install -e .
1.4 Check that the installation is successful
# 检查 Pytorch
import torch, torchvision
print('Pytorch 版本', torch.__version__)
print('CUDA 是否可用',torch.cuda.is_available())
# 检查 mmcv
import mmcv
from mmcv.ops import get_compiling_cuda_version, get_compiler_version
print('MMCV版本', mmcv.__version__)
print('CUDA版本', get_compiling_cuda_version())
print('编译器版本', get_compiler_version())
# 检查 mmagic
import mmagic
print('MMagic版本', mmagic.__version__)
1.5 Install other tool packages
!pip install opencv-python pillow matplotlib seaborn tqdm -i https://pypi.tuna.tsinghua.edu.cn/simple
!pip install clip transformers gradio 'httpx[socks]' diffusers==0.14.0 -i https://pypi.tuna.tsinghua.edu.cn/simple
!mim install 'mmdet>=3.0.0'
2.Application: Colorizing black and white photos
2.1 Enter the MMagic home directory
import os
os.chdir('mmagic')
2.2 Download sample images
!wget https://zihao-openmmlab.obs.cn-east-3.myhuaweicloud.com/20230613-MMagic/data/test_colorization.jpg -O test_colorization.jpg
2.3 Run prediction
!python demo/mmagic_inference_demo.py \
--model-name inst_colorization \
--img test_colorization.jpg \
--result-out-dir out_colorization.png
3. Application: Vincent Chart Stable Diffusion
3.1 Import toolkit
from mmagic.apis import MMagicInferencer
3.2 Load model
sd_inferencer = MMagicInferencer(model_name='stable_diffusion')
3.3 Specify Prompt text
text_prompts = 'A panda is having dinner at KFC'
text_prompts = 'A Persian cat walking in the streets of New York'
3.4 Forecast
sd_inferencer.infer(text=text_prompts, result_out_dir='output/sd_res.png')
4. Application: Vincent Picture Dreambooth
4.1 Enter the MMagic home directory
import os
os.chdir('mmagic')
4.2 Training Dreambooth on the dataset
!bash tools/dist_train.sh configs/dreambooth/dreambooth-lora.py 1
4.3 Use the trained model to make predictions
from mmengine import Config
from mmagic.registry import MODELS
from mmagic.utils import register_all_modules
register_all_modules()
cfg = Config.fromfile('./mmagic/configs/dreambooth/dreambooth-lora.py')
dreambooth_lora = MODELS.build(cfg.model)
state = torch.load('mmagic/work_dirs/dreambooth-lora/iter_1000.pth')['state_dict']
def convert_state_dict(state):
state_dict_new = {
}
for k, v in state.items():
if '.module' in k:
k_new = k.replace('.module', '')
else:
k_new = k
if 'vae' in k:
if 'to_q' in k:
k_new = k.replace('to_q', 'query')
elif 'to_k' in k:
k_new = k.replace('to_k', 'key')
elif 'to_v' in k:
k_new = k.replace('to_v', 'value')
elif 'to_out' in k:
k_new = k.replace('to_out.0', 'proj_attn')
state_dict_new[k_new] = v
return state_dict_new
dreambooth_lora.load_state_dict(convert_state_dict(state))
dreambooth_lora = dreambooth_lora.cuda()
samples = dreambooth_lora.infer('side view of sks dog', guidance_scale=5)
samples['samples'][0]
samples = dreambooth_lora.infer('ear close-up of sks dog', guidance_scale=5)
samples['samples'][0]
5. Application: Tushengtu-ControlNet-Canny
5.1 Enter the MMagic home directory
import os
os.chdir('mmagic')
5.2 Import toolkit
import cv2
import numpy as np
import mmcv
from mmengine import Config
from PIL import Image
from mmagic.registry import MODELS
from mmagic.utils import register_all_modules
register_all_modules()
5.3 Load ControlNet model
cfg = Config.fromfile('configs/controlnet/controlnet-canny.py')
controlnet = MODELS.build(cfg.model).cuda()
5.4 Input Canny edge map
control_url = 'https://user-images.githubusercontent.com/28132635/230288866-99603172-04cb-47b3-8adb-d1aa532d1d2c.jpg'
control_img = mmcv.imread(control_url)
control = cv2.Canny(control_img, 100, 200)
control = control[:, :, None]
control = np.concatenate([control] * 3, axis=2)
control = Image.fromarray(control)
5.5 Prompt
prompt = 'Room with blue walls and a yellow ceiling.'
5.6 Perform predictions
output_dict = controlnet.infer(prompt, control=control)
samples = output_dict['samples']
for idx, sample in enumerate(samples):
sample.save(f'sample_{
idx}.png')
controls = output_dict['controls']
for idx, control in enumerate(controls):
control.save(f'control_{
idx}.png')
6. Application: Tushengtu-ControlNet-Pose
6.1 Enter the MMagic home directory
import os
os.chdir('mmagic')
6.2 Import toolkit
import mmcv
from mmengine import Config
from PIL import Image
from mmagic.registry import MODELS
from mmagic.utils import register_all_modules
register_all_modules()
6.3 Load ControlNet model
cfg = Config.fromfile('configs/controlnet/controlnet-pose.py')
# convert ControlNet's weight from SD-v1.5 to Counterfeit-v2.5
cfg.model.unet.from_pretrained = 'gsdf/Counterfeit-V2.5'
cfg.model.vae.from_pretrained = 'gsdf/Counterfeit-V2.5'
cfg.model.init_cfg['type'] = 'convert_from_unet'
controlnet = MODELS.build(cfg.model).cuda()
# call init_weights manually to convert weight
controlnet.init_weights()
6.4 Prompt
prompt = 'masterpiece, best quality, sky, black hair, skirt, sailor collar, looking at viewer, short hair, building, bangs, neckerchief, long sleeves, cloudy sky, power lines, shirt, cityscape, pleated skirt, scenery, blunt bangs, city, night, black sailor collar, closed mouth'
6.5 Input Pose diagram
control_url = 'https://user-images.githubusercontent.com/28132635/230380893-2eae68af-d610-4f7f-aa68-c2f22c2abf7e.png'
control_img = mmcv.imread(control_url)
control = Image.fromarray(control_img)
control.save('control.png')
6.6 Executing predictions
output_dict = controlnet.infer(prompt, control=control, width=512, height=512, guidance_scale=7.5)
samples = output_dict['samples']
for idx, sample in enumerate(samples):
sample.save(f'sample_{
idx}.png')
controls = output_dict['controls']
for idx, control in enumerate(controls):
control.save(f'control_{
idx}.png')
7. Graph-ControlNet Animation
7.1 Method 1: Gradio command line
!python demo/gradio_controlnet_animation.py
7.2 Method 2: MMagic API
# 导入工具包
from mmagic.apis import MMagicInferencer
# Create a MMEdit instance and infer
editor = MMagicInferencer(model_name='controlnet_animation')
# 指定 prompt 咒语
prompt = 'a girl, black hair, T-shirt, smoking, best quality, extremely detailed'
negative_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
# 待测视频
# https://user-images.githubusercontent.com/12782558/227418400-80ad9123-7f8e-4c1a-8e19-0892ebad2a4f.mp4
video = '../run_forrest_frames_rename_resized.mp4'
save_path = '../output_video.mp4'
# 执行预测
editor.infer(video=video, prompt=prompt, image_width=512, image_height=512, negative_prompt=negative_prompt, save_path=save_path)