微信自动跳一跳 总结备忘

1.
电脑操作手机使用 WebDriverAgent.
从 https://github.com/facebook/WebDriverAgent clone 后 ，按照 github 上的说明，可以正确生成 xcode 工程.
WebDriverAgent的 xcode 工程依赖 Carthage 和 node 两个库，可以用 brew 安装


2.
WebDriverAgent xcode 工程设置成功后，打开 WebDriverAgent.xcodeproj
把里面 的 WebDriverAgentRunner target 用 Product -> Test 的方式 安装到手机上,并运行 app


过程中可能需要 设置自己的开发账号


工程部署好了之后，可以通过 xcode 命令行 自动 build WebDriverAgent 到 手机上
xcodebuild -project WebDriverAgent.xcodeproj -scheme WebDriverAgentRunner -destination "id=$(idevice_id -l)" test




3.
WebDriverAgentRunner 部署到 手机上成功后 ，用 brew 安装 usbuxd库，因为要使用 iproxy 命令远程操作手机


4.
使用 iproxy 8100 8100 把 手机的 8100 端口，映射到电脑的 8100 端口上。
然后访问 电脑 http://localhost:8100 等同于访问手机的 8100 端口 。浏览器上 打开 http://localhost:8100 如果有信息的话，电脑远程操作 手机的 准备工作就做好了。


5.
下载安装 python 的 atx 库。pip install atx
atx 给 python 提供了 各种直接操作手机的方法。
示例:


d = atx.connect('http://localhost:8100', platform='ios')
image = d.screenshot();
image.save("./testscreen.png");
d.click(200,200);
d.swipe(100,100,200,200,duration);


6.
python 下载cv2 库，使用 opencv 的功能
cv2 用于 分析截屏图片


使用示例


读取:
img = cv2.imread("./osizejump.jpg");


显示:
cv2.namedWindow("debug");
cv2.imshow("debug",img);
cv2.waitKey(0);


绘制(比如在图片上做标记):
cv2.namedWindow("debug");
cv2.rectangle(img,topLeft,bottomRight,(0,0,255),2);
cv2.imshow("debug",img);
cv2.waitKey(0);


图像匹配:
res = cv2.matchTemplate(screenImg,jumperTemp,cv2.TM_CCOEFF_NORMED);
min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res);


生成canny 图片(用于做边缘检测):
img = cv2.GaussianBlur(screenImg,(3,3),0); #500多分 的时候 都没用 高斯模糊
canny = cv2.Canny(img,35,100);


保存图片:
cv2.imwrite("./dest/temcanny.jpg",canny);


7.
自动跳跃思路:


(1) 使用 atx 远程操作手机 截屏，保存本地文件
(2) 使用 cv2 读取屏幕图片文件
(3) cv2.matchTemplate() 匹配 棋子，用于计算棋子位置
(4) cv2 生成 canny 图片用于做边缘检测
(5) 根据边缘检测图片，计算目标坐标点
(6) 根据棋子位置，和目标坐标点 ，计算距离，设置 长按时间
(7) 使用 atx swipe() 长按屏幕，完成自动跳跃


8.

代码没什么参考价值。写在这里做备忘

# -*- coding:utf-8 -*-


import atx
import cv2
import numpy as np
import math
import time

d = None;
d = atx.connect("http://localhost:8100");

screen = None;
screenImg = None;


jumperTemp = None;
jumperW = None;
jumperH = None;

targetTemp = None;
targetW = None;
targetH = None;


def debugImage(img):
	cv2.namedWindow("debug");
	cv2.imshow("debug",img);
	cv2.waitKey(0);

def debugRect(img,topLeft,bottomRight):
	cv2.namedWindow("debug");
	cv2.rectangle(img,topLeft,bottomRight,(0,0,255),2);
	cv2.imshow("debug",img);
	cv2.waitKey(0);

def debugPoint(img,x,y):
	cv2.namedWindow("debug");
	cv2.circle(img,(int(x),int(y)),10,(0,0,255),-1);
	cv2.imshow("debug",img);
	cv2.waitKey(0);

def initConst():
	global jumperTemp
	global jumperW
	global jumperH
	jumperTemp = cv2.imread("./osizejump.jpg");
	jumperW,jumperH = jumperTemp.shape[1],jumperTemp.shape[0];	

	global targetTemp
	global targetW
	global targetH
	targetTemp = cv2.imread("./osizetarget.jpg");
	targetW,targetH = targetTemp.shape[1],targetTemp.shape[0];

'''
src pos begin
'''
def genScreenshot():
	global screen
	global screenImg
	screen = d.screenshot();
	screen.save("./dest/screen.jpg");
	screenImg = cv2.imread("./dest/screen.jpg");

def matchJumper():
	global screenImg;
	global jumperTemp;
	global jumperW
	global jumperH
	res = cv2.matchTemplate(screenImg,jumperTemp,cv2.TM_CCOEFF_NORMED);
	# print(res);
	# debugImage(res);
	min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res);
	top_left = max_loc;
	bottom_right = (top_left[0] + jumperW,top_left[1] + jumperH);
	# debugRect(screenImg,top_left,bottom_right);

	x = top_left[0] + jumperW/2;
	y = top_left[1] + jumperH * 0.85;
	# debugPoint(screenImg,x,y);
	return x,y,min_loc,max_loc;

def getSrcPos():
	return matchJumper();
'''
src pos end
'''



'''
dest pos begin
'''

def get_center(img_canny, ):

	CHECK_HEIGHT = 100;

	START_CHECK_ROW = 400

	# debugImage(img_canny);
	theRow = np.nonzero([max(row) for row in img_canny[START_CHECK_ROW:]])

	# print("miao1 row:" + str(row));
	# print("miao1")
	# print(row);
	# print(pt);
	# print("miao1 end");

	# print("miao2")
	# print(theRow)
	# print("miao2 end")

	y_top = theRow[0][0] + START_CHECK_ROW
	# y_top = theRow[0][0]


	#@miao @test
	print("y_top:" + str(y_top))

	x_top = int(np.mean(np.nonzero(img_canny[y_top])))
	print("x_top:" + str(x_top));

	H = img_canny.shape[0];
	y_bottom = y_top + CHECK_HEIGHT
	print("y_bottom:" + str(y_bottom));
	for row in range(y_bottom, H):
		# print("check row:" + str(row));
		if img_canny[row, x_top] != 0:
			y_bottom = row
			break
	x_center, y_center = x_top, (y_top + y_bottom) // 2




	return img_canny, x_center, y_center


def detectEdge():
	global screenImg;

	img = cv2.GaussianBlur(screenImg,(3,3),0);	#500多分 的时候 都没用 高斯模糊
	img = screenImg;
	# debugImage(img);
	# canny = cv2.Canny(img,50,150);
	# canny = cv2.Canny(img,10,50);
	canny = cv2.Canny(img,35,100);
	# debugImage(canny);
	return canny;

def detectTarget():
	global screenImg;
	global targetTemp
	global targetW
	global targetH

	res = cv2.matchTemplate(screenImg,targetTemp,cv2.TM_CCOEFF_NORMED);
	# print(res);
	# debugImage(res);
	min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res);
	print("------- max_val:" + str(max_val));
	if max_val >= 0.8:
		print(" match target !");
		top_left = max_loc;
		bottom_right = (top_left[0] + targetW,top_left[1] + targetH);
		# debugRect(screenImg,top_left,bottom_right);
		x = top_left[0] + targetW/2;
		y = top_left[1] + targetH/2;

		# print(screenImg)
		# debugImage(screenImg);
		# print("x:" + str(x) + ",y:" + str(y))
		# debugPoint(screenImg,x,y);
		# print("miaomiao");
		return True,x,y;
	print("unmatch target");
	return False,0,0

def getDestPos(jumperMinLoc,jumperMaxLoc):
	global jumperW
	global jumperH
	global screenImg
	canny = detectEdge();
	# remove jumper in canny
	for k in range(jumperMaxLoc[1] - 10, jumperMaxLoc[1] + jumperH):
		for b in range(jumperMaxLoc[0] - 10, jumperMaxLoc[0] + jumperW):
			canny[k][b] = 0
	# debugImage(canny);
	cv2.imwrite("./dest/temcanny.jpg",canny);

	# print(canny)

	x = 0
	y = 0
	# if detectTarget():
	bResult,x,y = detectTarget();
	if bResult:
		print("got target spot");
		return x,y;
	else:
		canny,x,y = get_center(canny)
		print("find next center");
	# print(x)
	# print(y);
	# debugPoint(screenImg,x,y);
	return x,y

'''
dest pos end
'''


def playStep():
	global d
	global screenImg
	genScreenshot();
	x1,y1,minLoc,maxLoc = getSrcPos();
	# debugPoint(screenImg,x1,y1);
	x2,y2 = getDestPos(minLoc,maxLoc);
	# debugPoint(screenImg,x2,y2);
	print("src:("+str(x1) + ","+str(y1) + ")")
	print("dst:("+str(x1) + ","+str(y2) + ")")

	cv2.circle(screenImg,(int(x1),int(y1)),10,(0,0,255),-1);
	cv2.circle(screenImg,(int(x2),int(y2)),10,(255,0,0),-1);
	cv2.imwrite("./dest/fenxi.jpg",screenImg);

	distance = math.pow((x2 - x1),2) + math.pow((y2 - y1),2)
	distance = math.sqrt(distance)
	print("distance:"+str(distance));

	# duration = distance / 1047.1 	#score 431
	# duration = distance / 1046.8 	# score 554
	duration = distance / 1046.5
	print("duration:"+str(duration));


	if distance < 429 and distance > 300:
		duration = duration - 0.04
		print("fix 0 ..")
	elif distance <= 300:
		print("fix 1 .."); 
		duration = duration - 0.09;

	'''
	FAIL !!! @miao @todo
	dst:(519,982)
distance:191.095689381
duration:0.182552244346
fix 1 ..
fix 2...
final duration:0.1
^CTraceback (most recent call la


unmatch target
y_top:932
x_top:654
y_bottom:1032
find next center
src:(519,1117.25)
dst:(519,982)
distance:191.095689381
duration:0.182552244346
fix 1 ..
fix 2...
final duration:0.1
^CTraceback (most recent call last):
  File "dest2.py", line 258, in <module>
    playStep();

	'''

	if duration < 0.07:
		duration = 0.07;
		print("fix 2...");

	print("final duration:" + str(duration));

	d.swipe(100,100,200,200,duration);



initConst();

while True:
	time.sleep(0.9)
	playStep();

# playStep();

# test case
# genScreenshot();
# screenImg = cv2.imread("./dest/hastarget.jpg");
# detectTarget();
# matchJumper();