多线程处理数据案例——对比单线程和多线程处理数据的速度

一、功能说明

最近进行一个功能优化工作，涉及到数据的传输，由于散乱的小文件太多的缘故，因此在数据传输过程中非常慢。计划通过对文件夹进行压缩后传输二进制压缩包的方式提升网络传输效率。前期需要对某个目录下的所有文件夹逐个压缩为zip包，此处引入多线程处理和单线程处理，对比一下压缩速度。 代码并未提供全部，主要是为了记录多线程处理文件时参数分配以及进度界面显示问题。

二、功能界面

三、执行效果

四、主要功能代码

// MutiThreadDlg.h : 头文件
//

#pragma once
#include "afxcmn.h"
#include "Myprogressctrl.h"

#include <vector>
#include <map>
#include "afxwin.h"
using namespace std;

struct threadInfo  
{
    
      
	threadInfo()
	{
    
    
		nCodeNum = -1;
		pctrlProgress = NULL;
		ctrlTip = NULL;
		vecInfo.clear();
	}

	threadInfo::threadInfo(const threadInfo& other)
	{
    
    
		nCodeNum = other.nCodeNum;
		pctrlProgress = other.pctrlProgress;
		ctrlTip = other.ctrlTip;
		vecInfo.assign(other.vecInfo.begin(),other.vecInfo.end());
	}

	threadInfo& operator = (threadInfo& thInfo)
	{
    
    
		nCodeNum = thInfo.nCodeNum;
		pctrlProgress = thInfo.pctrlProgress;
		ctrlTip = thInfo.ctrlTip;
		vecInfo.assign(thInfo.vecInfo.begin(),thInfo.vecInfo.end());

		return* this;
	}

	~threadInfo()
	{
    
    

	}

	UINT nCodeNum;  
	std::vector<CString> vecInfo;
	CProgressCtrl* pctrlProgress;
	CStatic* ctrlTip;
};  

//DWORD WINAPI SingleThreadFunc(LPVOID lpParam);  
DWORD WINAPI MutiThreadFunc(LPVOID lpParam);  


// CMutiThreadDlg 对话框
class CMutiThreadDlg : public CDialogEx
{
    
    
// 构造
public:
	CMutiThreadDlg(CWnd* pParent = NULL);	// 标准构造函数

// 对话框数据
	enum {
    
     IDD = IDD_MUTITHREAD_DIALOG };

	protected:
	virtual void DoDataExchange(CDataExchange* pDX);	// DDX/DDV 支持

// 实现
protected:
	HICON m_hIcon;

	// 生成的消息映射函数
	virtual BOOL OnInitDialog();
	afx_msg void OnPaint();
	afx_msg HCURSOR OnQueryDragIcon();
	DECLARE_MESSAGE_MAP()

public:
	afx_msg void OnBnClickedBtnMuti();
	afx_msg void OnBnClickedBtnSingle();
	afx_msg void OnBnClickedBtnFilepath();

	// 从指定目录读取所有文件夹名称
	void GetAllFolder(std::string path, vector<std::string>& vecFiles);

	CMyProgressCtrl m_ctrlProgress;
	CStatic m_ctrlTip;
	std::vector<threadInfo> m_vecInfo;	
};

// MutiThreadDlg.cpp : 实现文件
//

#include "stdafx.h"
#include "MutiThread.h"
#include "MutiThreadDlg.h"
#include "afxdialogex.h"

#include "FileMiniZip.h"

#ifdef _DEBUG
#define new DEBUG_NEW
#endif

volatile UINT m_nFileCount = 0;
volatile UINT m_nOkCount = 0;
volatile long m_BeginTime = 0;

#define THRED_NUM  5

HANDLE m_hMutex; // 线程互斥

std::string wstring2string(wstring wstr)
{
    
    
	string result;
	int len = WideCharToMultiByte(CP_ACP, 0, wstr.c_str(), wstr.size(), NULL, 0, NULL, NULL);
	if( len <= 0 )return result;
	char* buffer = new char[len + 1];
	if(buffer == NULL )return result;
	WideCharToMultiByte(CP_ACP, 0, wstr.c_str(), wstr.size(), buffer, len, NULL, NULL);
	buffer[len] = '\0';
	result.append(buffer);
	delete[] buffer;

	return result;
}

std::wstring string2wstring(string str)
{
    
    
	wstring result;
	int len = MultiByteToWideChar(CP_ACP, 0, str.c_str(), str.size(), NULL, 0);
	if( len < 0 )return result;
	wchar_t* buffer = new wchar_t[len + 1];
	if( buffer == NULL )return result;
	MultiByteToWideChar(CP_ACP, 0, str.c_str(), str.size(), buffer, len);
	buffer[len] = '\0';
	result.append(buffer);
	delete[] buffer;

	return result;
}

bool UnZipPackage(PCTSTR szFile, PCTSTR szDestDir)
{
    
    
	std::wstring wsFile = szFile;
	std::string sZipFilePath = wstring2string(wsFile);

	std::wstring wsDestDir = szDestDir;
	std::string sUnZipToPath = wstring2string(wsDestDir);
	if (_tmkdir(wsDestDir.c_str()) != 0)
	{
    
    
		return false;
	}

	CFileMiniZip unzipTool;

	if (unzipTool.unZipPackageToLocal(sZipFilePath, sUnZipToPath) < 1)
	{
    
    
		return false;
	}

	return true;
}

bool ZipPackage(PCTSTR szSourcePath, PCTSTR szDestFile)
{
    
    
	CFileMiniZip unzipTool;

	std::wstring wsSourcePath = szSourcePath;
	std::string sSrcDirPath = wstring2string(wsSourcePath);

	std::wstring wsDestFile = szDestFile;
	std::string sZipFile = wstring2string(wsDestFile);

	if (unzipTool.CompressToPackageZip(sSrcDirPath, sZipFile) == 0)
	{
    
    
		return false;
	}

	return true;
}

DWORD WINAPI MutiThreadFunc(LPVOID lpParam)
{
    
    
	threadInfo* pInfo = (threadInfo*)lpParam;  
	int nCodeNum = pInfo->nCodeNum;
	std::vector<CString>& vecFileame = pInfo->vecInfo;

	for(int i = 0;i < vecFileame.size(); i++)  
	{
    
      
		CString strFolder = vecFileame[i];
		CString strFolderZip = strFolder + _T(".zip");

		CString sMsg;
		sMsg.Format(_T("DEBUG:当前线程序号= %d，包含文件数量= %d，处理文件名称= %s, !"),nCodeNum,(int)vecFileame.size(),vecFileame[i]);
		OutputDebugString(sMsg);

		// 测试压缩
		if (!ZipPackage(strFolder,strFolderZip))
		{
    
    
			OutputDebugString(_T("DEBUG::压缩失败！"));
		}

		WaitForSingleObject(m_hMutex, INFINITE);

		++m_nOkCount;
		long dCurrentTime = GetTickCount64();
		double dTime = ((double)(dCurrentTime - m_BeginTime))/1000;
		CString sTip;
		sTip.Format(_T("当前处理完成个数 %d / %d，耗时 %.3f 秒,剩余 %d 个!"),m_nOkCount,m_nFileCount,dTime,m_nFileCount-m_nOkCount);
		pInfo->ctrlTip->SetWindowTextW(sTip);
		pInfo->pctrlProgress->SetPos(m_nOkCount);

		ReleaseMutex(m_hMutex);
	}  

	return 0;  
}

// CMutiThreadDlg 对话框

CMutiThreadDlg::CMutiThreadDlg(CWnd* pParent /*=NULL*/)
	: CDialogEx(CMutiThreadDlg::IDD, pParent)
{
    
    
	m_hIcon = AfxGetApp()->LoadIcon(IDR_MAINFRAME);
}

void CMutiThreadDlg::DoDataExchange(CDataExchange* pDX)
{
    
    
	CDialogEx::DoDataExchange(pDX);
	DDX_Control(pDX, IDC_PROGRESS, m_ctrlProgress);
	DDX_Control(pDX, IDC_STATIC_TEXT, m_ctrlTip);
}

BEGIN_MESSAGE_MAP(CMutiThreadDlg, CDialogEx)
	ON_WM_PAINT()
	ON_WM_QUERYDRAGICON()
	ON_BN_CLICKED(IDC_BTN_MUTI, &CMutiThreadDlg::OnBnClickedBtnMuti)
	ON_BN_CLICKED(IDC_BTN_SINGLE, &CMutiThreadDlg::OnBnClickedBtnSingle)
	ON_BN_CLICKED(IDC_BTN_FILEPATH, &CMutiThreadDlg::OnBnClickedBtnFilepath)
END_MESSAGE_MAP()


// CMutiThreadDlg 消息处理程序

BOOL CMutiThreadDlg::OnInitDialog()
{
    
    
	CDialogEx::OnInitDialog();

	// 设置此对话框的图标。当应用程序主窗口不是对话框时，框架将自动
	//  执行此操作
	SetIcon(m_hIcon, TRUE);			// 设置大图标
	SetIcon(m_hIcon, FALSE);		// 设置小图标
	CString strFolderPath = _T("C:\\Users\\...\\Desktop\\..\\data");
	GetDlgItem(IDC_EDIT_FILEPATH)->SetWindowTextW(strFolderPath);

	return TRUE;  // 除非将焦点设置到控件，否则返回 TRUE
}

// 如果向对话框添加最小化按钮，则需要下面的代码
//  来绘制该图标。对于使用文档/视图模型的 MFC 应用程序，
//  这将由框架自动完成。

void CMutiThreadDlg::OnPaint()
{
    
    
	if (IsIconic())
	{
    
    
		CPaintDC dc(this); // 用于绘制的设备上下文

		SendMessage(WM_ICONERASEBKGND, reinterpret_cast<WPARAM>(dc.GetSafeHdc()), 0);

		// 使图标在工作区矩形中居中
		int cxIcon = GetSystemMetrics(SM_CXICON);
		int cyIcon = GetSystemMetrics(SM_CYICON);
		CRect rect;
		GetClientRect(&rect);
		int x = (rect.Width() - cxIcon + 1) / 2;
		int y = (rect.Height() - cyIcon + 1) / 2;

		// 绘制图标
		dc.DrawIcon(x, y, m_hIcon);
	}
	else
	{
    
    
		CDialogEx::OnPaint();
	}
}

//当用户拖动最小化窗口时系统调用此函数取得光标
//显示。
HCURSOR CMutiThreadDlg::OnQueryDragIcon()
{
    
    
	return static_cast<HCURSOR>(m_hIcon);
}

void CMutiThreadDlg::OnBnClickedBtnSingle()
{
    
    
	// TODO: 在此添加控件通知处理程序代码
	OutputDebugString(_T("DEBUG:开始执行单线程压缩"));

	// 获取目录下所有文件夹
	CString strDir;
	GetDlgItem(IDC_EDIT_FILEPATH)->GetWindowTextW(strDir);
	std::wstring wsDir = strDir;
	std::string sDir = wstring2string(wsDir);

	vector<std::string> vecFolders;
	GetAllFolder(sDir,vecFolders);
	if (vecFolders.size() == 0)
	{
    
    
		MessageBox(_T("所选目录为空！"));
		return;
	}

	m_vecInfo.clear();

	// 创建线程数据参数
	threadInfo thInfo;
	thInfo.nCodeNum = 0;
	thInfo.pctrlProgress = &m_ctrlProgress; 
	thInfo.ctrlTip = &m_ctrlTip;
	m_vecInfo.push_back(thInfo);

	m_nFileCount = (UINT)vecFolders.size();
	m_ctrlProgress.SetPos(0);
	m_ctrlProgress.SetRange(0,m_nFileCount);

	for (unsigned int n = 0; n < vecFolders.size(); ++n)
	{
    
    
		string sTemp = vecFolders[n];
		wstring wsTemp = string2wstring(sTemp);
		CString strTemp = (LPCTSTR)wsTemp.c_str();
		m_vecInfo[0].vecInfo.push_back(strTemp);
	}

	// 防止多次点击创建多个锁
	if (m_hMutex)
	{
    
    
		CloseHandle(m_hMutex);
		m_hMutex = NULL;
	}

	m_hMutex = CreateMutex(NULL, FALSE, _T("CompressPro"));
	m_BeginTime = GetTickCount64();

	threadInfo& sInfoTemp = m_vecInfo.at(0);
	DWORD ThreadID;
	HANDLE hThread = CreateThread(NULL,0,(LPTHREAD_START_ROUTINE)MutiThreadFunc,&sInfoTemp,0,&ThreadID);
	CloseHandle(hThread);
}

void CMutiThreadDlg::OnBnClickedBtnMuti()
{
    
    
	// TODO: 在此添加控件通知处理程序代码
	OutputDebugString(_T("DEBUG:开始执行多线程压缩"));

	// 获取目录下所有文件夹
	CString strDir;
	GetDlgItem(IDC_EDIT_FILEPATH)->GetWindowTextW(strDir);
	std::wstring wsDir = strDir;
	std::string sDir = wstring2string(wsDir);

	vector<std::string> vecFolders;
	GetAllFolder(sDir,vecFolders);
	if (vecFolders.size() == 0)
	{
    
    
		MessageBox(_T("所选目录为空！"));
		return;
	}
	
	m_vecInfo.clear();
	m_vecInfo.reserve(THRED_NUM);

	// 创建线程数据参数数组
	for (UINT n = 0; n < THRED_NUM; ++n)
	{
    
    
		threadInfo thInfo;
		thInfo.nCodeNum = n;
		thInfo.pctrlProgress = &m_ctrlProgress; 
		thInfo.ctrlTip = &m_ctrlTip;
		m_vecInfo.push_back(thInfo);
	}

	m_nFileCount = (UINT)vecFolders.size();
	m_ctrlProgress.SetPos(0);
	m_ctrlProgress.SetRange(0,m_nFileCount);

	UINT nFileNum = 0;
	for (unsigned int n = 0; n < vecFolders.size(); ++n)
	{
    
    
		string sTemp = vecFolders[n];
		wstring wsTemp = string2wstring(sTemp);
		CString strTemp = (LPCTSTR)wsTemp.c_str();
		m_vecInfo[nFileNum].vecInfo.push_back(strTemp);
		++nFileNum;

		if(nFileNum > THRED_NUM-1)
		{
    
    
			nFileNum = 0;
		}
	}

	// 防止多次点击创建多个锁
	if (m_hMutex)
	{
    
    
		CloseHandle(m_hMutex);
		m_hMutex = NULL;
	}

	m_hMutex = CreateMutex(NULL, FALSE, _T("CompressPro"));
	m_BeginTime = GetTickCount64();

	for (int i = 0; i < THRED_NUM; ++i)
	{
    
    
		threadInfo& sInfoTemp = m_vecInfo.at(i);
		DWORD ThreadID;
		HANDLE hThread = CreateThread(NULL,0,(LPTHREAD_START_ROUTINE)MutiThreadFunc,&sInfoTemp,0,&ThreadID);
		CloseHandle(hThread);
	}
}

void CMutiThreadDlg::OnBnClickedBtnFilepath()
{
    
    
	// TODO: 在此添加控件通知处理程序代码
	TCHAR szFolderPath[MAX_PATH] = {
    
    0};
	CString strFolderPath;
	BROWSEINFO sInfo;
	ZeroMemory(&sInfo, sizeof(BROWSEINFO));

	sInfo.pidlRoot = 0;
	sInfo.lpszTitle = _T("请选择一个文件夹：");
	sInfo.ulFlags = BIF_DONTGOBELOWDOMAIN | BIF_RETURNONLYFSDIRS | BIF_EDITBOX;
	sInfo.lpfn = NULL;

	// 显示文件夹选择对话框
	LPITEMIDLIST lpidlBrowse = SHBrowseForFolder(&sInfo);
	if (lpidlBrowse != NULL)
	{
    
    
		// 取得文件夹名
		if (SHGetPathFromIDList(lpidlBrowse,szFolderPath))
		{
    
    
			strFolderPath = szFolderPath;
		}
	}
	if(lpidlBrowse != NULL)
	{
    
    
		CoTaskMemFree(lpidlBrowse);
	}

	GetDlgItem(IDC_EDIT_FILEPATH)->SetWindowTextW(strFolderPath);
}

void CMutiThreadDlg::GetAllFolder(std::string path, vector<std::string>& vecFolders)
{
    
    
	// 文件句柄
	intptr_t hFile = 0;

	// 文件信息
	struct _finddata_t fileinfo;  

	string p;
	if ((hFile = _findfirst(p.assign(path).append("\\*").c_str(), &fileinfo)) != -1) 
	{
    
    
		do 
		{
    
    
			// 保存文件的全路径
			char* pPath = fileinfo.name;
			if ( strcmp(pPath,".") == 0 || strcmp(pPath,"..") == 0 )
			{
    
    
				continue;
			}

			vecFolders.push_back(p.assign(path).append("\\").append(fileinfo.name));

		} while (_findnext(hFile, &fileinfo) == 0);  //寻找下一个，成功返回0，否则-1

		_findclose(hFile);
	}
}

下载地址：https://download.csdn.net/download

注意：未加入界面控件保护，在执行过程可能误触，建议使用时自己加入界面保护。