1. Put it in front
Recently, the front-end buddy asked me a problem, asking me to implement a picture-to-MP4 video conversion with music on the back-end, and then provide an interface for him to download. Although I haven’t done related functions, I will just copy it In principle, I still bit the bullet and agreed...
Without further ado, let's get started!
2. Introduce dependencies
First of all, I searched the Internet for relevant demos, and I found them.
Here is the original link https://cloud.tencent.com/developer/article/1640244
Introduce related dependencies
<!--图片转MP4-->
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>javacv</artifactId>
<version>1.5.6</version>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>ffmpeg-platform</artifactId>
<version>4.4-1.5.6</version>
</dependency>
new class
package cn.hjljy.javacv;
import org.bytedeco.ffmpeg.global.avcodec;
import org.bytedeco.ffmpeg.global.avutil;
import org.bytedeco.javacv.*;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
/**
* @author 海加尔金鹰 www.hjljy.cn
* @version V1.0
* @email [email protected]
* @description: 图片合成MP4
* @since 2020/5/16 18:00
**/
public class Image2Mp4 {
public static void main(String[] args) throws Exception {
//合成的MP4
String mp4SavePath = "D:\\javacv\\mp4\\img.mp4";
//图片地址 这里面放了22张图片
String img = "D:\\javacv\\img";
int width = 1600;
int height = 900;
//读取所有图片
File file = new File(img);
File[] files = file.listFiles();
Map<Integer, File> imgMap = new HashMap<Integer, File>();
int num = 0;
for (File imgFile : files) {
imgMap.put(num, imgFile);
num++;
}
createMp4(mp4SavePath, imgMap, width, height);
}
private static void createMp4(String mp4SavePath, Map<Integer, File> imgMap, int width, int height) throws FrameRecorder.Exception {
//视频宽高最好是按照常见的视频的宽高 16：9 或者 9：16
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(mp4SavePath, width, height);
//设置视频编码层模式
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
//设置视频为25帧每秒
recorder.setFrameRate(25);
//设置视频图像数据格式
recorder.setPixelFormat(avutil.AV_PIX_FMT_YUV420P);
recorder.setFormat("mp4");
try {
recorder.start();
Java2DFrameConverter converter = new Java2DFrameConverter();
//录制一个22秒的视频
for (int i = 0; i < 22; i++) {
BufferedImage read = ImageIO.read(imgMap.get(i));
//一秒是25帧 所以要记录25次
for (int j = 0; j < 25; j++) {
recorder.record(converter.getFrame(read));
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//最后一定要结束并释放资源
recorder.stop();
recorder.release();
}
}
}
then blend music
public static boolean mergeAudioAndVideo(String videoPath, String audioPath, String outPut) throws Exception {
boolean isCreated = true;
File file = new File(videoPath);
if (!file.exists()) {
return false;
}
FrameRecorder recorder = null;
FrameGrabber grabber1 = null;
FrameGrabber grabber2 = null;
try {
//抓取视频帧
grabber1 = new FFmpegFrameGrabber(videoPath);
//抓取音频帧
grabber2 = new FFmpegFrameGrabber(audioPath);
grabber1.start();
grabber2.start();
//创建录制
recorder = new FFmpegFrameRecorder(outPut,
grabber1.getImageWidth(), grabber1.getImageHeight(),
grabber2.getAudioChannels());
recorder.setFormat("mp4");
recorder.setFrameRate(grabber1.getFrameRate());
recorder.setSampleRate(grabber2.getSampleRate());
recorder.start();
Frame frame1;
Frame frame2 ;
//先录入视频
while ((frame1 = grabber1.grabFrame()) != null ){
recorder.record(frame1);
}
//然后录入音频
while ((frame2 = grabber2.grabFrame()) != null) {
recorder.record(frame2);
}
grabber1.stop();
grabber2.stop();
recorder.stop();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (recorder != null) {
recorder.release();
}
if (grabber1 != null) {
grabber1.release();
}
if (grabber2 != null) {
grabber2.release();
}
} catch (FrameRecorder.Exception e) {
e.printStackTrace();
}
}
return isCreated;
}
3. Start the test
I started to write a demo based on the demo on the Internet, but if there is no accident, there will be an accident...
The picture has indeed been converted to an MP4 video, which is also 1 second, but the generated video looks like a red filter is added...
After a pass through Baidu and the source code (of course I can’t understand it), I finally found a solution on StackOverflow, just change the parsing mode, and n times if it doesn’t work once
(attach my writing method)
//生成MP4视频
private String createMp4(String mp4SavePath, List<File> list, int width, int height) throws Exception {
String mp3Name = "";
//视频宽高最好是按照常见的视频的宽高 16：9 或者 9：16
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(mp4SavePath, width, height);
// 文件格式
recorder.setFormat("mp4");
// 帧率与抓取器一致
recorder.setFrameRate(25);
// 编码器类型
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
try {
recorder.start();
Java2DFrameConverter converter = new Java2DFrameConverter();
for (File file : list) {
if (file.getName().contains(".mp3")) {
mp3Name = file.getName();
continue;
}
BufferedImage read = null;
try {
read = ImageIO.read(file);
//循环的数量等于帧率 保证每秒一张图片。
for (int i = 0; i < recorder.getFrameRate(); i++) {
//编码格式 ##这里解决生成视频颜色不对问题
recorder.record(converter.getFrame(read), avutil.AV_PIX_FMT_RGB32_1);
}
} catch (IOException e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//最后一定要结束并释放资源
recorder.stop();
recorder.release();
}
return mp3Name;
}
In the online demo, recorder.setPixelFormat(avutil.AV_PIX_FMT_YUV420P);
I borrowed (copied) the master of StackOverflow to specify the encoding format when parsing each picture:
recorder.record(converter.getFrame(read), avutil.AV_PIX_FMT_RGB32_1);
4. Fusion Music
Okay, on to the next step, dubbing...
However, according to the tips of this article:
1 The length of the video and the length of the audio should be as consistent as possible. If they are inconsistent, the length of the synthesized video will be the longest one. If the audio is short, the audio will be missing later. If the video is short, the subsequent video will show the last frame of the video.
2 It is not recommended to record a frame of video and then record a frame of audio, the second half of the audio will be lost, the ratio is almost 1:1.6! ! !
Indeed, as this big brother wrote, the inconsistent length of video and audio will cause problems in the generated MP4, so I quickly solved this problem with my clever brain...
5. Complete example
5.1 Control layer
//上传图片转MP4
@PostMapping("/upload/mp4/transfer")
public ResponseEntity<InputStreamResource> uploadMP4(@RequestParam("files") MultipartFile[] files) throws Exception {
return imgManagerService.uploadMP4(files);
}
5.2 Business layer
//将图片转为MP4图片
@Override
public ResponseEntity<InputStreamResource> uploadMP4(MultipartFile[] files) throws Exception {
log.info("=======生成MP4开始========");
String uuid = IdGen.uuid();
for (MultipartFile file : files) {
getPathAndSaveFile(file, uuid);//上传到服务器的指定目录
}
// 图片集合的目录
String imagesPath = rootPath + uuid + "/";
String saveMp4name = imagesPath + "source.mp4"; //保存的视频名称
List<File> list = FileUtil.readFile(imagesPath);
int width = 1849;
int height = 932;
//返回MP3名字
String mp3 = createMp4(saveMp4name, list, width, height);
String outPutName = imagesPath + uuid + ".mp4";
mergeAudioAndVideo(saveMp4name, StringUtils.isBlank(mp3) ? null : imagesPath + mp3, outPutName);
log.info("=======生成MP4结束========");
File file = new File(outPutName);
InputStreamResource inputStreamResource = new InputStreamResource(new FileInputStream(file));
return ResponseEntity.ok()
.header("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"")
.contentType(MediaType.valueOf("audio/mp4"))
.body(inputStreamResource);
}
The methods involved:
1.getPathAndSaveFile(file, uuid) #file is the uploaded file #uuid is the identification of a single request
private String getPathAndSaveFile(MultipartFile file, String taskId) throws IOException {
String path = rootPath + taskId + "/" + file.getOriginalFilename();
File newFile = new File(path);
FileUtils.copyInputStreamToFile(file.getInputStream(), newFile);
return path;
}
2.FileUtil.readFile(imagesPath) #imagesPath The image path saved to the server
/**
* 读取文件
*
* @param imgUrl 文件路径
* @throws IOException
*/
public static List<File> readFile(String imgUrl) throws IOException {
//读取所有图片
File file = new File(imgUrl);
return Arrays.stream(Objects.requireNonNull(file.listFiles())).collect(Collectors.toList());
}
3. The method to generate MP4 video is createMp4(saveMp4name, list, width, height)
#saveMp4name Saved video name
#list file list
#width height The width and height of the generated video should be consistent with the picture
//生成MP4视频
private String createMp4(String mp4SavePath, List<File> list, int width, int height) throws Exception {
String mp3Name = "";
//视频宽高最好是按照常见的视频的宽高 16：9 或者 9：16
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(mp4SavePath, width, height);
// 文件格式
recorder.setFormat("mp4");
// 帧率与抓取器一致
recorder.setFrameRate(25);
// 编码器类型
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
try {
recorder.start();
Java2DFrameConverter converter = new Java2DFrameConverter();
for (File file : list) {
if (file.getName().contains(".mp3")) {
mp3Name = file.getName();
continue;
}
BufferedImage read = null;
try {
read = ImageIO.read(file);
//循环的数量等于帧率 保证每秒一张图片。
for (int i = 0; i < recorder.getFrameRate(); i++) {
//编码格式
recorder.record(converter.getFrame(read), avutil.AV_PIX_FMT_RGB32_1);
}
} catch (IOException e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//最后一定要结束并释放资源
recorder.stop();
recorder.release();
}
return mp3Name;
}
4. Dubbing...
mergeAudioAndVideo(saveMp4name, StringUtils.isBlank(mp3) ? null : imagesPath + mp3, outPutName);
saveMp4name #MP4 video to be operated
StringUtils.isBlank(mp3) ? null : imagesPath + mp3 #The MP3 music to be operated will pass null if there is no MP3
outPutName #output directory
//配音
public void mergeAudioAndVideo(String videoPath, String audioPath, String outPut) throws Exception {
//没有音频文件 直接返回原文件
if (null == audioPath){
new File(videoPath).renameTo(new File(outPut));
return;
}
FFmpegFrameRecorder recorder = null;
FrameGrabber grabber1 = null;
FrameGrabber grabber2 = null;
try {
//抓取视频帧
grabber1 = new FFmpegFrameGrabber(videoPath);
//抓取音频帧
grabber2 = new FFmpegFrameGrabber(audioPath);
grabber1.start();
grabber2.start();
//创建录制
recorder = new FFmpegFrameRecorder(outPut,
grabber1.getImageWidth(), grabber1.getImageHeight(),
grabber2.getAudioChannels());
recorder.setFormat("mp4");
recorder.setFrameRate(grabber1.getFrameRate());
recorder.setSampleRate(grabber2.getSampleRate());
// 视频质量，0表示无损
recorder.setVideoQuality(0);
recorder.start();
Frame frame1;
Frame frame2 = null;
//先录入视频
while ((frame1 = grabber1.grabFrame()) != null) {
recorder.record(frame1);
}
//然后录入音频
audioEntry(frame2, grabber1, grabber2, recorder);
grabber1.stop();
grabber2.stop();
recorder.stop();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (recorder != null) {
recorder.release();
}
if (grabber1 != null) {
grabber1.release();
}
if (grabber2 != null) {
grabber2.release();
}
} catch (FrameRecorder.Exception e) {
e.printStackTrace();
}
}
}
5. How to record audio
audioEntry(frame2, grabber1, grabber2, recorder);
#frame2 audio frame
Information for the #grabber1 video
#grabber2 music message
#recorder Action Tool
private void audioEntry(Frame frame, FrameGrabber grabber1, FrameGrabber grabber2, FrameRecorder recorder) throws Exception {
long grabber1Timestamp = grabber1.getTimestamp();
while ((frame = grabber2.grabFrame()) != null) {
//如果视频时长小于音频时长 则截断音频帧
if (grabber1Timestamp <= grabber2.getTimestamp()) break;
recorder.record(frame);
}
long differ = grabber1Timestamp - grabber2.getTimestamp();
//如果视频时长大于音频时长 则循环录入
if (differ > 0) {
grabber1.setTimestamp(differ);
audioEntry(frame, grabber1, grabber2, recorder);
}
}
6. Write at the end
In this way, when this interface is written, the front-end friends directly praise me for 666. Of course, there is still room for optimization of the code here...
1. Whether the decoders for generating MP4 video and dubbing can be shared (of course I didn't think of it)
2. If there are multiple audio files, you need to modify the code again
3. No matter what, go fishing
7. Record
Using the previous method to generate video will cause the video to be unplayable, so update the code
//生成MP4视频
private void createMp4(String mp4SavePath, List<File> list, int width, int height) throws Exception {
String mp3Name = "";
//视频宽高最好是按照常见的视频的宽高 16：9 或者 9：16
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(mp4SavePath, width, height);
//MP4格式设置成H264，才能在html上播放
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
// 设置格式mp4
recorder.setFormat("mp4");
// 此处说明每一秒多少帧，即说明1秒会录多少张照片
recorder.setFrameRate(1); //0.01 代表100秒一张图
// 8000kb/s 这个说明视频每秒大小，值越大图片转过来的压缩率就越小质量就会越高
recorder.setVideoBitrate(80000);//80000000
// yuv420p
recorder.setPixelFormat(avutil.AV_PIX_FMT_YUV420P);
// 先默认吧，这个应该属于设置视频的处理模式 不可变(固定)音频比特率
recorder.setAudioOption("crf", "0");
// 最高质量
recorder.setAudioQuality(0);
// 音频比特率
recorder.setAudioBitrate(192000);
// 音频采样率
recorder.setSampleRate(44100);
// 双通道(立体声)
recorder.setAudioChannels(2);
// ------------------->end 初始化视频录制器
try {
recorder.start();// 开始录制
// ------------------->begin 图片处理开始
OpenCVFrameConverter.ToIplImage conveter = new OpenCVFrameConverter.ToIplImage(); // 申明一个图片处理的变量
for (File file : list) {
if (file.getName().contains(".mp3")) {
mp3Name = file.getPath();
continue;
}
IplImage image = opencv_imgcodecs.cvLoadImage(file.getPath()); // 非常吃内存！！
Frame frame = conveter.convert(image);
recorder.record(frame); // 录制
// 释放内存 非常吃内存！！
opencv_core.cvReleaseImage(image);
}
// ------------------->end 图片处理开始
// ------------------->begin 开始录制音频
if (StringUtils.isNotBlank(mp3Name)) {
FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(mp3Name);
grabber.start();// 开始录制音频
audioEntry(null, grabber, recorder, recorder.getTimestamp());
// ------------------->end 开始录制音频
grabber.stop();
grabber.release();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//最后一定要结束并释放资源
recorder.stop();
recorder.release();
}
}
//录入音频 mp3Grabber 音频帧 frame音频源 recorder 解码器 recorderTimestamp 视频总长
private void audioEntry(Frame frame, FFmpegFrameGrabber mp3Grabber, FFmpegFrameRecorder recorder, long recorderTimestamp) throws Exception {
while ((frame = mp3Grabber.grabFrame()) != null) {
//如果视频时长小于音频时长 则截断音频帧
long timestamp = mp3Grabber.getTimestamp();
if (recorderTimestamp <= mp3Grabber.getTimestamp()) break;
recorder.record(frame);
}
long differ = recorderTimestamp - mp3Grabber.getTimestamp();
mp3Grabber.setTimestamp(0);
//如果视频时长大于音频时长 则循环录入
if (differ > 0) {
audioEntry(frame, mp3Grabber, recorder, differ);
}
}