Dependencies: C# OpenCVSharp WPF Numpy
Purpose: Solve the position coordinates of the characters projected onto the two-dimensional plane on the football field
Figure A/B/C
1. Basic concepts
1.1 Definition of standard court:
参考：https://zh.m.wikipedia.org/zh/%E8%B6%B3%E7%90%83%E5%A0%B4
Figure D
1.2 Registration ideas
Figure A -> Figure B, establish a coordinate relationship pair;
Figure B-> Figure D, establish the registration reference point pairs of the real coordinate system (requires more than 4)
Figure D -> Figure C, establish the registration reference point pair of the display coordinate system (the size of Figure C is known, and all key positions AJ are known)
For the global situation, use OpenCV FindHomography to obtain the DC coordinate transformation matrix T;
For each frame of picture, use OpenCV FindHomography to obtain the BD coordinate transformation matrix T1;
Project object detected players to Figure 3
2. Code implementation
2.1 Mark key points
WPF interface
<Grid>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="0.4*"/>
<ColumnDefinition Width="0.6*"/>
</Grid.ColumnDefinitions>
<Grid x:Name="gridPointSelector" Width="350" Height="234" VerticalAlignment="Center" HorizontalAlignment="Center" >
<Grid.ColumnDefinitions>
<ColumnDefinition Width="0.15619*"/>
<ColumnDefinition Width="0.34762*"/>
<ColumnDefinition Width="0.34762*"/>
<ColumnDefinition Width="0.15619*"/>
</Grid.ColumnDefinitions>
<Grid.RowDefinitions>
<RowDefinition Height="0.20441*" />
<RowDefinition Height="0.59118*"/>
<RowDefinition Height="0.20441*"/>
</Grid.RowDefinitions>
<Image Source="{StaticResource 2D_field}" Grid.ColumnSpan="4" Grid.RowSpan="3" />
<ToggleButton Content="A" Grid.Row="0" Grid.Column="0"
HorizontalAlignment="Left" VerticalAlignment="Top" Margin="-15,-15,0,0" Click="ToggleButton_Click"/>
<ToggleButton Content="B" Grid.Row="0" Grid.Column="1"
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0" Click="ToggleButton_Click"/>
<ToggleButton Content="C" Grid.Row="0" Grid.Column="3"
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0" Click="ToggleButton_Click"/>
<ToggleButton Content="D" Grid.Row="1" Grid.Column="0"
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0" Click="ToggleButton_Click"/>
<ToggleButton Content="E" Grid.Row="1" Grid.Column="2"
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0" Click="ToggleButton_Click"/>
<ToggleButton Content="F" Grid.Row="2" Grid.Column="0"
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,-15,-15,0" Click="ToggleButton_Click"/>
<ToggleButton Content="G" Grid.Row="2" Grid.Column="2"
HorizontalAlignment="Right" VerticalAlignment="Top" Click="ToggleButton_Click"/>
<ToggleButton Content="H" Grid.Row="2" Grid.Column="0"
HorizontalAlignment="Left" VerticalAlignment="Bottom" Click="ToggleButton_Click" />
<ToggleButton Content="I" Grid.Row="2" Grid.Column="1"
HorizontalAlignment="Right" VerticalAlignment="Bottom" Click="ToggleButton_Click"/>
<ToggleButton Content="J" Grid.Row="2" Grid.Column="3"
HorizontalAlignment="Right" VerticalAlignment="Bottom" Click="ToggleButton_Click"/>
</Grid>
<Grid Grid.Column="1" x:Name="gridMain" ClipToBounds="True">
<Image x:Name="imgMain" Source="{Binding OriginalImage }" Stretch="Uniform"/>
<Canvas x:Name="canvas" Height="{Binding ElementName=imgMain,Path=ActualHeight}"
Width="{Binding ElementName=imgMain,Path=ActualWidth}" Background="Transparent"
IsEnabled="{Binding IsSelectting}" MouseLeftButtonUp="canvas_MouseLeftButtonUp">
</Canvas>
</Grid>
</Grid>
data structure
public class VedioPointMark
{
public string? Name {
get; set; }
/// <summary>
/// 球场坐标
/// </summary>
public Point2d FiledPoint {
get; set; }
/// <summary>
/// 视频像素坐标
/// </summary>
public Point2d VideoPixelPoint {
get; set; }
}
WPF backend code
public partial class UCPointMark : UserControl
{
MPointMark model;
private readonly int ellipseSize = 16;
ToggleButton? selectButton;
public UCPointMark()
{
InitializeComponent();
model = new MPointMark();
this.DataContext = model;
}
private Border? DrawPoint()
{
if (selectButton == null)
return null;
Border myBorder = new Border();
myBorder.CornerRadius = new CornerRadius(ellipseSize);
myBorder.Width = ellipseSize;
myBorder.BorderBrush = new SolidColorBrush(Colors.Blue);
myBorder.Background = new SolidColorBrush(Colors.Red);
myBorder.Child = new TextBlock() {
Text = selectButton.Content.ToString(),
Foreground = new SolidColorBrush(Colors.White),
TextAlignment = TextAlignment.Center};
selectButton.Tag = myBorder;
return myBorder;
}
public List<VedioPointMark> CalculateTransform()
{
var markPoints = new List<VedioPointMark>();
//必须选择4各以上
if (canvas.Children.Count < 4)
return markPoints;
var allPoints = VideoFieldTransform.CreatePointMarks();
var axisX = (double)model.OriginalImageSize.Width / this.imgMain.ActualWidth;
var axisY = (double)model.OriginalImageSize.Height / this.imgMain.ActualHeight;
foreach (var element in this.gridPointSelector.Children)
{
if (element is ToggleButton toggle && toggle.Tag is Border pixcelBorder
&& pixcelBorder.Tag is Point pixcelPoint && toggle.IsChecked == true)
{
var mark = allPoints.Where(p => p.Name?.Equals(toggle.Content.ToString()) == true).FirstOrDefault();
if (mark == null)
continue;
mark.VideoPixelPoint = new OpenCvSharp.Point2d( pixcelPoint.X * axisX, pixcelPoint.Y * axisY -40 );
markPoints.Add(mark);
}
}
return markPoints;
}
private void canvas_MouseLeftButtonUp(object sender, MouseButtonEventArgs e)
{
try
{
var clickPoint = e.GetPosition(this.canvas);
var ellipse = DrawPoint();
if (null == ellipse) return;
this.canvas.Children.Add(ellipse);
Canvas.SetLeft(ellipse, clickPoint.X - ellipseSize / 2);
Canvas.SetTop(ellipse, clickPoint.Y - ellipseSize / 2);
ellipse.Tag = new Point(clickPoint.X, clickPoint.Y);
model.IsSelectting = false;
}
catch (Exception ex)
{
this.Log(ex);
}
}
2.2 Registration Transformation Class
public class VideoFieldTransform
{
private readonly Point2d[] point1; //视频可视范围内的关键点坐标（4个以上）
private readonly Point2d[] point2; //真实球场坐标系 或 平面坐标系
private readonly Mat H; //变换矩阵
public VideoFieldTransform(List<VedioPointMark> pointMarks,bool haspoint2 = false)
{
if (!haspoint2)
{
//视频坐标投影到物理坐标
pointMarks = pointMarks.OrderBy(p => p.Name).ToList();
point1 = pointMarks.Select(p => p.FiledPoint).ToArray();
point2 = pointMarks.Select(p => p.VideoPixelPoint).ToArray();
}
else
{
//物理坐标投影到图片
point1 = new Point2d[] {
new Point2d(0, 0),
new Point2d(16.4f, 13.9f),
new Point2d(52.5f, 0.0f),
new Point2d(88.6f, 13.9f ),
new Point2d(105f, 0.0f),
new Point2d(105f, 68.0f),
new Point2d(52.5f, 68f),
new Point2d(0, 68f)
};
point2 = new Point2d[] {
new Point2d(0.0f, 0.0f ),
new Point2d(164f, 152f ),
new Point2d( 525f, 0.0f),
new Point2d(886f, 152f),
new Point2d(1050f, 0f),
new Point2d(1050f, 699f),
new Point2d(525f, 699f),
new Point2d(0f, 699f),
};
}
H = CalculateHomoGraphy(haspoint2);
}
public static List<VedioPointMark> CreatePointMarks()
{
return new List<VedioPointMark>()
{
new VedioPointMark() {
Name = "A", FiledPoint = new Point2d(0, 0)},
new VedioPointMark() {
Name = "D", FiledPoint = new Point2d(16.4f, 13.9f)},
new VedioPointMark() {
Name = "B", FiledPoint = new Point2d(52.5f, 0.0f)},
new VedioPointMark() {
Name = "E", FiledPoint = new Point2d(88.6f, 13.9f)},
new VedioPointMark() {
Name = "C", FiledPoint = new Point2d(105f, 0.0f)},
new VedioPointMark() {
Name = "J", FiledPoint = new Point2d(105f, 68.0f)},
new VedioPointMark() {
Name = "I", FiledPoint = new Point2d(52.5f, 68f)},
new VedioPointMark() {
Name = "H", FiledPoint = new Point2d(0, 68f)},
new VedioPointMark() {
Name = "F", FiledPoint = new Point2d(16.4f, 40.2f)},
new VedioPointMark() {
Name = "G", FiledPoint = new Point2d(88.6, 40.2f)},
};
}
/// <summary>
/// 视频帧(假定摄像头位置不变)与 平面模式的球场位置的坐标系换算，求得矩阵
/// </summary>
/// <param name="make_rotate"></param>
/// <returns>返回H 为变换矩阵</returns>
private Mat CalculateHomoGraphy(bool make_rotate = false)
{
//var k = InputArray.Create(point2.GetData<float[]>());
if (!make_rotate)
return Cv2.FindHomography(point2, point1, HomographyMethods.Ransac);
return Cv2.FindHomography(point1,point2, HomographyMethods.Ransac);
}
/// <summary>
/// 根据坐标点(X,Y) 与 坐标系变换矩阵乘积，换算帧图像的位置到平面球场坐标的位置
/// </summary>
public Point Transform(Point p)
{
var img2Bounds = new[]
{
new Point2d(p.X, p.Y)
};
var img2BoundsTransformed = Cv2.PerspectiveTransform(img2Bounds, H);
var drawingPoints = img2BoundsTransformed.Select(p => (Point)p).FirstOrDefault();
return drawingPoints;
}
}
2.3 Initialization
Figure D -> Figure C, establish the registration reference point pair of the display coordinate system (the size of Figure C is known, and all key positions AJ are known)
Among them, pointMarks is the set of points marked by WPF
if (pointMarks?.Count > 3)
proj_field_to_top = new VideoFieldTransform(pointMarks); //真实球场坐标
For conversion from absolute pitch size to display size, see 1.2 DC Transformation
proj_field_2d = new VideoFieldTransform(true); //平面显示坐标
load basemap
//加载球场真实坐标系底图
D_field_photo = LoadImages.Load("2D_field.png");
Initialize object detection
//对象检测,此处略...参见AI机器学习（五）相关的内容
detector = new DetectorYolov7();
2.4 Frame-by-frame transformation
Assuming a player's Mat area, the pixel position in the video is marked as rect
//获取帧图片
foreach(var frame in LoadImages.LoadVideo("你的视频路径"))
{
Mat imagedetect = new Mat();
//投影所有的球员
foreach(var prediction =detector.Detect(imagedetect))
{
var rect = prediction.Rectangle;
//拷贝真实球场坐标底图
var J = D_field_photo.Clone();
//转换坐标系
var pfield = proj_field_to_top.Transform(new OpenCvSharp.Point(rect.X , rect.Y));
//用于显示的图像
var pshow = proj_field_2d.Transform(new OpenCvSharp.Point(pfield.X, pfield.Y));
Cv2.Circle(J, pshow.X, pshow.Y, 10, Scalar.LightCyan, -1);
//发送给前端WPF，这里没给出用于显示的页面，大家自己实现一个即可，只需要一个Image控件
OnDisplay?.Invoke(null, J.Resize(new OpenCvSharp.Size(300, 200)).ToBytes());
}
}