本次实验我们要实现的是以平面和标记物进行姿态估计和增强现实。
在实现代码之前，我们需要安装两个工具包
1.PyGame，下载网址为 https://www.lfd.uci.edu/~gohlke/pythonlibs/#pygame ；
2.OpenGL，下载网址为 https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyopengl 。
关于安装OpenGL
到以上网址里去下载，根据自己电脑的版本及Python版本选择下载。
PS：cp27是对应python2.7版本的，我的python是3.7版本的，所以选择最下面的进行下载。
然后打开命令窗口，通过 pip install PyOpenGl-3.1.3b2-cp27m-win_amd64.whl 来安装opengl。（记得在文件下在哪个盘上就在哪个盘上进行安装）
然后我们需要打开 python\Lib\site-packages\OpenGL\DLLS 这个文件夹，留下glut64.vc15.dll这个文件，其他删除。
关于安装PyGame
PyGame可以直接pip install pygame安装。
一、以平面和标记物进行姿态估计
如果图像中包含平面状的标记物体，并且以及对照相机进行了标定，那么我们可以计算出照相机的姿态（旋转或平移）。这里的标记物体可以为对任何平坦的物体。
以下实验是使用平面物体作为标记物，来计算用于新视图投影矩阵的例子。
代码实现：
from pylab import *
from PIL import Image
# If you have PCV installed, these imports should work
from PCV.geometry import homography, camera
from PCV.localdescriptors import sift
"""
This is the augmented reality and pose estimation cube example from Section 4.3.
"""
#创建用于绘制立方体的一个点列表（前5个点是底部的正方形，一些边重合了）
def cube_points(c, wid):
""" Creates a list of points for plotting
a cube with plot. (the first 5 points are
the bottom square, some sides repeated). """
p = []
# 底部
p.append([c[0]-wid, c[1]-wid, c[2]-wid])
p.append([c[0]-wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]-wid, c[2]-wid])
p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot
# 顶部
p.append([c[0]-wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot
# 竖直边
p.append([c[0]-wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]-wid])
return array(p).T
def my_calibration(sz):
"""
Calibration function for the camera (iPhone4) used in this example.
"""
row, col = sz
fx = 2555*col/2592
fy = 2586*row/1936
K = diag([fx, fy, 1])
K[0, 2] = 0.5*col
K[1, 2] = 0.5*row
return K
# 计算特征
sift.process_image('book1.jpg', 'im0.sift')
l0, d0 = sift.read_features_from_file('im0.sift')
sift.process_image('book2.jpg', 'im1.sift')
l1, d1 = sift.read_features_from_file('im1.sift')
# 匹配特征并计算单应性矩阵
matches = sift.match_twosided(d0, d1)
ndx = matches.nonzero()[0]
fp = homography.make_homog(l0[ndx, :2].T)
ndx2 = [int(matches[i]) for i in ndx]
tp = homography.make_homog(l1[ndx2, :2].T)
model = homography.RansacModel()
H, inliers = homography.H_from_ransac(fp, tp, model)
# 计算照相机标定矩阵
K = my_calibration((747, 1000))
# 位于边长为0.2 z=0平面的三维点
box = cube_points([0, 0, 0.1], 0.1)
# 投影第一幅图像上底部的正方形
cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
# 底部正方形上的点
box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
# 使用H将点变换到第二幅图像中
box_trans = homography.normalize(dot(H,box_cam1))
# 从cam1和H中计算第二个照相机矩阵
cam2 = camera.Camera(dot(H, cam1.P))
A = dot(linalg.inv(K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(K, A)
# 使用第二个照相机矩阵投影
box_cam2 = cam2.project(homography.make_homog(box))
# plotting
im0 = array(Image.open('book1.jpg'))
im1 = array(Image.open('book2.jpg'))
figure()
imshow(im0)
plot(box_cam1[0, :], box_cam1[1, :], linewidth=3)
title('2D projection of bottom square')
axis('off')
figure()
imshow(im1)
plot(box_trans[0, :], box_trans[1, :], linewidth=3)
title('2D projection transfered with H')
axis('off')
figure()
imshow(im1)
plot(box_cam2[0, :], box_cam2[1, :], linewidth=3)
title('3D points projected in second image')
axis('off')
show()
运行结果：
我们使用的图像的分辨率为747*1000。将图像的特征和对齐后的标记匹配，计算出单应性矩阵，然后用于计算照相机的姿态。
二、增强现实
增强现实是将物体和相应信息放置在图像数据上的一系列操作的总称。代码实现了放置一个三维计算机图形学模型，使其看起来属于该场景。以下实验我们在书本的平面上叠加了者茶壶。
代码实现：
import math
import pickle
from pylab import *
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *
import pygame, pygame.image
from pygame.locals import *
from PCV.geometry import homography, camera
from PCV.localdescriptors import sift
def cube_points(c, wid):
""" Creates a list of points for plotting
a cube with plot. (the first 5 points are
the bottom square, some sides repeated). """
p = []
# bottom
p.append([c[0]-wid, c[1]-wid, c[2]-wid])
p.append([c[0]-wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]-wid, c[2]-wid])
p.append([c[0]-wid, c[1]-wid, c[2]-wid]) #same as first to close plot
# top
p.append([c[0]-wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]-wid, c[2]+wid]) #same as first to close plot
# vertical sides
p.append([c[0]-wid, c[1]-wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]+wid])
p.append([c[0]-wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]-wid])
p.append([c[0]+wid, c[1]+wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]+wid])
p.append([c[0]+wid, c[1]-wid, c[2]-wid])
return array(p).T
def my_calibration(sz):
row, col = sz
fx = 2555*col/2592
fy = 2586*row/1936
K = diag([fx, fy, 1])
K[0, 2] = 0.5*col
K[1, 2] = 0.5*row
return K
def set_projection_from_camera(K):
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
fx = K[0,0]
fy = K[1,1]
fovy = 2*math.atan(0.5*height/fy)*180/math.pi
aspect = (width*fy)/(height*fx)
near = 0.1
far = 100.0
gluPerspective(fovy,aspect,near,far)
glViewport(0,0,width,height)
def set_modelview_from_camera(Rt):
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
Rx = np.array([[1,0,0],[0,0,-1],[0,1,0]])
R = Rt[:,:3]
U,S,V = np.linalg.svd(R)
R = np.dot(U,V)
R[0,:] = -R[0,:]
t = Rt[:,3]
M = np.eye(4)
M[:3,:3] = np.dot(R,Rx)
M[:3,3] = t
M = M.T
m = M.flatten()
glLoadMatrixf(m)
def draw_background(imname):
bg_image = pygame.image.load(imname).convert()
bg_data = pygame.image.tostring(bg_image,"RGBX",1)
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glEnable(GL_TEXTURE_2D)
glBindTexture(GL_TEXTURE_2D,glGenTextures(1))
glTexImage2D(GL_TEXTURE_2D,0,GL_RGBA,width,height,0,GL_RGBA,GL_UNSIGNED_BYTE,bg_data)
glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_NEAREST)
glBegin(GL_QUADS)
glTexCoord2f(0.0,0.0); glVertex3f(-1.0,-1.0,-1.0)
glTexCoord2f(1.0,0.0); glVertex3f( 1.0,-1.0,-1.0)
glTexCoord2f(1.0,1.0); glVertex3f( 1.0, 1.0,-1.0)
glTexCoord2f(0.0,1.0); glVertex3f(-1.0, 1.0,-1.0)
glEnd()
glDeleteTextures(1)
def draw_teapot(size):
glEnable(GL_LIGHTING)
glEnable(GL_LIGHT0)
glEnable(GL_DEPTH_TEST)
glClear(GL_DEPTH_BUFFER_BIT)
glMaterialfv(GL_FRONT,GL_AMBIENT,[0,0,0,0])
glMaterialfv(GL_FRONT,GL_DIFFUSE,[0.5,0.0,0.0,0.0])
glMaterialfv(GL_FRONT,GL_SPECULAR,[0.7,0.6,0.6,0.0])
glMaterialf(GL_FRONT,GL_SHININESS,0.25*128.0)
glutSolidTeapot(size)
width,height = 1000,747
def setup():
pygame.init()
pygame.display.set_mode((width,height),OPENGL | DOUBLEBUF)
pygame.display.set_caption("OpenGL AR demo")
# compute features
sift.process_image('book1.jpg', 'im0.sift')
l0, d0 = sift.read_features_from_file('im0.sift')
sift.process_image('book2.jpg', 'im1.sift')
l1, d1 = sift.read_features_from_file('im1.sift')
# match features and estimate homography
matches = sift.match_twosided(d0, d1)
ndx = matches.nonzero()[0]
fp = homography.make_homog(l0[ndx, :2].T)
ndx2 = [int(matches[i]) for i in ndx]
tp = homography.make_homog(l1[ndx2, :2].T)
model = homography.RansacModel()
H, inliers = homography.H_from_ransac(fp, tp, model)
K = my_calibration((747, 1000))
cam1 = camera.Camera(hstack((K, dot(K, array([[0], [0], [-1]])))))
box = cube_points([0, 0, 0.1], 0.1)
box_cam1 = cam1.project(homography.make_homog(box[:, :5]))
box_trans = homography.normalize(dot(H,box_cam1))
cam2 = camera.Camera(dot(H, cam1.P))
A = dot(linalg.inv(K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(K, A)
Rt=dot(linalg.inv(K),cam2.P)
setup()
draw_background("book2.jpg")
set_projection_from_camera(K)
set_modelview_from_camera(Rt)
draw_teapot(0.05)
pygame.display.flip()
while True:
for event in pygame.event.get():
if event.type==pygame.QUIT:
sys.exit()
运行结果：
在实现以上两个实验，碰上了以下问题
问题一：
解决办法：要正确选择相对应的python版本，并在相对应的盘上安装。
问题二：
出现错误freeglut ERROR: Function called without first calling ‘glutInit’. 这个错误是freeglut和glut共存的缘故，它们俩定义了相同的方法，这是动态链接库的重叠问题。
解决办法：我们需要打开 python\Lib\site-packages\OpenGL\DLLS 这个文件夹，留下glut64.vc15.dll这个文件就好了，其他删除。
