ffplay播放器剖析(5)----视频输出剖析

1.视频输出模块

1.1 视频输出初始化

1.1.1 视频输出初始化主要流程

1. 初始化SDL,SDL_Init,主要是SDL_INIT_VIDEO的支持
2. SDL_CreateWindow,创建主窗口
3. SDL_CreateRender,基于主窗口创建renderer,用于渲染输出
4. stream_open
5. event_loop,播放控制事件的相应循环,但也负责了video的显示输出

int main(int argc, char **argv)
{
    
    
   
    /* 是否显示视频 */
    if (display_disable) {
    
    
        video_disable = 1;
    }
    // 3. SDL的初始化
    flags = SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER;
    /* 是否运行音频 */
    if (audio_disable)
        flags &= ~SDL_INIT_AUDIO;
    else {
    
    
        /* Try to work around an occasional ALSA buffer underflow issue when the
         * period size is NPOT due to ALSA resampling by forcing the buffer size. */
        if (!SDL_getenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE"))
            SDL_setenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE","1", 1);
    }
    if (display_disable)
        flags &= ~SDL_INIT_VIDEO;
    if (SDL_Init (flags)) {
    
    
        av_log(NULL, AV_LOG_FATAL, "Could not initialize SDL - %s\n", SDL_GetError());
        av_log(NULL, AV_LOG_FATAL, "(Did you set the DISPLAY variable?)\n");
        exit(1);
    }

    SDL_EventState(SDL_SYSWMEVENT, SDL_IGNORE);
    SDL_EventState(SDL_USEREVENT, SDL_IGNORE);

    av_init_packet(&flush_pkt);				// 初始化flush_packet
    flush_pkt.data = (uint8_t *)&flush_pkt; // 初始化为数据指向自己本身

    // 4. 创建窗口
    if (!display_disable) {
    
    
        int flags = SDL_WINDOW_HIDDEN;
        if (alwaysontop)
#if SDL_VERSION_ATLEAST(2,0,5)
            flags |= SDL_WINDOW_ALWAYS_ON_TOP;
#else
            av_log(NULL, AV_LOG_WARNING, "Your SDL version doesn't support SDL_WINDOW_ALWAYS_ON_TOP. Feature will be inactive.\n");
#endif
        if (borderless)
            flags |= SDL_WINDOW_BORDERLESS;
        else
            flags |= SDL_WINDOW_RESIZABLE;
        window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);
        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
        if (window) {
    
    
            // 创建renderer
            renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
            if (!renderer) {
    
    
                av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError());
                renderer = SDL_CreateRenderer(window, -1, 0);
            }
            if (renderer) {
    
    
                if (!SDL_GetRendererInfo(renderer, &renderer_info))
                    av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name);
            }
        }
        if (!window || !renderer || !renderer_info.num_texture_formats) {
    
    
            av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
            do_exit(NULL);
        }
    }
    // 5. 通过stream_open函数，开启read_thread读取线程
    is = stream_open(input_filename, file_iformat);
    if (!is) {
    
    
        av_log(NULL, AV_LOG_FATAL, "Failed to initialize VideoState!\n");
        do_exit(NULL);
    }

    // 6. 事件响应
    event_loop(is);

    /* never returns */

    return 0;
}

    //7 从待处理流中获取相关参数，设置显示窗口的宽度、高度及宽高比
    if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {
    
    
        AVStream *st = ic->streams[st_index[AVMEDIA_TYPE_VIDEO]];
        AVCodecParameters *codecpar = st->codecpar;
        //根据流和帧宽高比猜测视频帧的像素宽高比（像素的宽高比，注意不是图像的）
        AVRational sar = av_guess_sample_aspect_ratio(ic, st, NULL);
        if (codecpar->width) {
    
    
            // 设置显示窗口的大小和宽高比
            set_default_window_size(codecpar->width, codecpar->height, sar);
        }
    }

这里重点讲解一下set_default_window_size函数

static void set_default_window_size(int width, int height, AVRational sar)
{
    
    
    SDL_Rect rect;
    int max_width  = screen_width  ? screen_width  : INT_MAX; // 确定是否指定窗口最大宽度
    int max_height = screen_height ? screen_height : INT_MAX; // 确定是否指定窗口最大高度
    if (max_width == INT_MAX && max_height == INT_MAX)
        max_height = height;    // 没有指定最大高度时则使用视频的高度
    calculate_display_rect(&rect, 0, 0, max_width, max_height, width, height, sar);
    default_width  = rect.w; // 实际是渲染区域的宽高
    default_height = rect.h;
}

screen_width和screen_height可以在ffplay启动时用命令行进行设置 -x -y 如果没有指定那么就使用视频帧的高度

重点就是calculate_display_rect函数!

1.1.2 calculate_display_rect初始化显示窗口大小

static void calculate_display_rect(SDL_Rect *rect,
                                   int scr_xleft, int scr_ytop, int scr_width, int scr_height,
                                   int pic_width, int pic_height, AVRational pic_sar)
{
    
    
    AVRational aspect_ratio = pic_sar; // 比率
    int64_t width, height, x, y;

    if (av_cmp_q(aspect_ratio, av_make_q(0, 1)) <= 0)
        aspect_ratio = av_make_q(1, 1);// 如果aspect_ratio是负数或者为0,设置为1:1
    // 转成真正的播放比例
    aspect_ratio = av_mul_q(aspect_ratio, av_make_q(pic_width, pic_height));

    /* XXX: we suppose the screen has a 1.0 pixel ratio */
    // 计算显示视频帧区域的宽高
    // 先以高度为基准
    height = scr_height;
    // &~1, 取偶数宽度  1110
    width = av_rescale(height, aspect_ratio.num, aspect_ratio.den) & ~1;
    if (width > scr_width) {
    
    
        // 当以高度为基准,发现计算出来的需要的窗口宽度不足时调整为以窗口宽度为基准
        width = scr_width;
        height = av_rescale(width, aspect_ratio.den, aspect_ratio.num) & ~1;
    }
    // 计算显示视频帧区域的起始坐标（在显示窗口内部的区域）
    x = (scr_width - width) / 2;
    y = (scr_height - height) / 2;
    rect->x = scr_xleft + x;
    rect->y = scr_ytop  + y;
    rect->w = FFMAX((int)width,  1);
    rect->h = FFMAX((int)height, 1);
}

这个函数设置了计算了窗口的宽高和位置

函数是先计算宽高比,如果宽高比没有设置的话则使用实际宽高来计算宽高比,然后先以高度为基准使用av_rescale函数计算宽度,如果宽度大于scr_width就转变为以宽度为基准.

然后就是计算顶点坐标了,scr_width和scr_height窗口大小,width和heigth是视频大小,我们要计算出视频的左上角位置

看图:

代码中rect就是渲染的视频部分,也就是图中绿色部分!

1.2 视频输出逻辑

main()->
    event_loop()->
    refresh_loop_wait_event()->
    video_refresh()->
    video_display()->
    video_image_display()->
    upload_texture()

1.2.1 event_loop开始处理SDL事件

static void event_loop(VideoState *cur_stream)
{
    
    
    SDL_Event event;
    double incr, pos, frac;

    for (;;) {
    
    
        double x;
        refresh_loop_wait_event(cur_stream, &event); //video是在这里显示的
        switch (event.type) {
    
    
        case SDL_KEYDOWN:	/* 键盘事件 */
            if (exit_on_keydown || event.key.keysym.sym == SDLK_ESCAPE || event.key.keysym.sym == SDLK_q) {
    
    
                do_exit(cur_stream);
                break;
            }
            if (!cur_stream->width)
                continue;
            switch (event.key.keysym.sym) {
    
    
            case SDLK_f:
                toggle_full_screen(cur_stream);
                cur_stream->force_refresh = 1;
                break;
            case SDLK_p:
            case SDLK_SPACE: //按空格键触发暂停/恢复
                toggle_pause(cur_stream);
                break;
            case SDLK_m:
                toggle_mute(cur_stream);
                break;
            case SDLK_KP_MULTIPLY:
            case SDLK_0:
                update_volume(cur_stream, 1, SDL_VOLUME_STEP);
                break;
            case SDLK_KP_DIVIDE:
            case SDLK_9:
                update_volume(cur_stream, -1, SDL_VOLUME_STEP);
                break;
            case SDLK_s: // S: Step to next frame
                step_to_next_frame(cur_stream);
                break;
            case SDLK_a:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
                break;
            case SDLK_v:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
                break;
            case SDLK_c:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
                break;
            case SDLK_t:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
                break;
            case SDLK_w:
#if CONFIG_AVFILTER
                if (cur_stream->show_mode == SHOW_MODE_VIDEO && cur_stream->vfilter_idx < nb_vfilters - 1) {
    
    
                    if (++cur_stream->vfilter_idx >= nb_vfilters)
                        cur_stream->vfilter_idx = 0;
                } else {
    
    
                    cur_stream->vfilter_idx = 0;
                    toggle_audio_display(cur_stream);
                }
#else
                toggle_audio_display(cur_stream);
#endif
                break;
            case SDLK_PAGEUP:
                if (cur_stream->ic->nb_chapters <= 1) {
    
    
                    incr = 600.0;
                    goto do_seek;
                }
                seek_chapter(cur_stream, 1);
                break;
            case SDLK_PAGEDOWN:
                if (cur_stream->ic->nb_chapters <= 1) {
    
    
                    incr = -600.0;
                    goto do_seek;
                }
                seek_chapter(cur_stream, -1);
                break;
            case SDLK_LEFT:
                incr = seek_interval ? -seek_interval : -10.0;
                goto do_seek;
            case SDLK_RIGHT:
                incr = seek_interval ? seek_interval : 10.0;
                goto do_seek;
            case SDLK_UP:
                incr = 60.0;
                goto do_seek;
            case SDLK_DOWN:
                incr = -60.0;
            do_seek:
                if (seek_by_bytes) {
    
    
                    pos = -1;
                    if (pos < 0 && cur_stream->video_stream >= 0)
                        pos = frame_queue_last_pos(&cur_stream->pictq);
                    if (pos < 0 && cur_stream->audio_stream >= 0)
                        pos = frame_queue_last_pos(&cur_stream->sampq);
                    if (pos < 0)
                        pos = avio_tell(cur_stream->ic->pb);
                    if (cur_stream->ic->bit_rate)
                        incr *= cur_stream->ic->bit_rate / 8.0;
                    else
                        incr *= 180000.0;
                    pos += incr;
                    stream_seek(cur_stream, pos, incr, 1);
                } else {
    
    
                    pos = get_master_clock(cur_stream);
                    if (isnan(pos))
                        pos = (double)cur_stream->seek_pos / AV_TIME_BASE;
                    pos += incr;    // 现在是秒的单位
                    if (cur_stream->ic->start_time != AV_NOPTS_VALUE && pos < cur_stream->ic->start_time / (double)AV_TIME_BASE)
                        pos = cur_stream->ic->start_time / (double)AV_TIME_BASE;
                    stream_seek(cur_stream, (int64_t)(pos * AV_TIME_BASE), (int64_t)(incr * AV_TIME_BASE), 0);
                }
                break;
            default:
                break;
            }
            break;
        case SDL_MOUSEBUTTONDOWN:			/* 鼠标按下事件 */
            if (exit_on_mousedown) {
    
    
                do_exit(cur_stream);
                break;
            }
            if (event.button.button == SDL_BUTTON_LEFT) {
    
    
                static int64_t last_mouse_left_click = 0;
                if (av_gettime_relative() - last_mouse_left_click <= 500000) {
    
    
                    //连续鼠标左键点击2次显示窗口间隔小于0.5秒，则进行全屏或者恢复原始窗口
                    toggle_full_screen(cur_stream);
                    cur_stream->force_refresh = 1;
                    last_mouse_left_click = 0;
                } else {
    
    
                    last_mouse_left_click = av_gettime_relative();
                }
            }
        case SDL_MOUSEMOTION:		/* 鼠标移动事件 */
            if (cursor_hidden) {
    
    
                SDL_ShowCursor(1);
                cursor_hidden = 0;
            }
            cursor_last_shown = av_gettime_relative();
            if (event.type == SDL_MOUSEBUTTONDOWN) {
    
    
                if (event.button.button != SDL_BUTTON_RIGHT)
                    break;
                x = event.button.x;
            } else {
    
    
                if (!(event.motion.state & SDL_BUTTON_RMASK))
                    break;
                x = event.motion.x;
            }
            if (seek_by_bytes || cur_stream->ic->duration <= 0) {
    
    
                uint64_t size =  avio_size(cur_stream->ic->pb); // 整个文件的字节
                stream_seek(cur_stream, size*x/cur_stream->width, 0, 1);
            } else {
    
    
                int64_t ts;
                int ns, hh, mm, ss;
                int tns, thh, tmm, tss;
                tns  = cur_stream->ic->duration / 1000000LL;
                thh  = tns / 3600;
                tmm  = (tns % 3600) / 60;
                tss  = (tns % 60);
                frac = x / cur_stream->width;
                ns   = frac * tns;
                hh   = ns / 3600;
                mm   = (ns % 3600) / 60;
                ss   = (ns % 60);
                av_log(NULL, AV_LOG_INFO,
                       "Seek to %2.0f%% (%2d:%02d:%02d) of total duration (%2d:%02d:%02d)       \n", frac*100,
                       hh, mm, ss, thh, tmm, tss);
                ts = frac * cur_stream->ic->duration;
                if (cur_stream->ic->start_time != AV_NOPTS_VALUE)
                    ts += cur_stream->ic->start_time;
                stream_seek(cur_stream, ts, 0, 0);
            }
            break;
        case SDL_WINDOWEVENT:		/* 窗口事件 */
            switch (event.window.event) {
    
    
            case SDL_WINDOWEVENT_SIZE_CHANGED:
                screen_width  = cur_stream->width  = event.window.data1;
                screen_height = cur_stream->height = event.window.data2;
                if (cur_stream->vis_texture) {
    
    
                    SDL_DestroyTexture(cur_stream->vis_texture);
                    cur_stream->vis_texture = NULL;
                }
            case SDL_WINDOWEVENT_EXPOSED:
                cur_stream->force_refresh = 1;
            }
            break;
        case SDL_QUIT:
        case FF_QUIT_EVENT:	/* ffplay自定义事件,用于主动退出 */
            do_exit(cur_stream);
            break;
        default:
            break;
        }
    }
}

这个函数主要是通过refresh_loop_wait_event函数等待事件,然后event_loop进行处理事件.

扫描二维码关注公众号，回复： 16984357 查看本文章

video的显示主要在refresh_loop_wait_event中:

static void refresh_loop_wait_event(VideoState *is, SDL_Event *event) {
    
    
    double remaining_time = 0.0; /* 休眠等待，remaining_time的计算在video_refresh中 */
    /* 调用SDL_PeepEvents前先调用SDL_PumpEvents，将输入设备的事件抽到事件队列中 */
    SDL_PumpEvents();
    /*
     * SDL_PeepEvents check是否事件，比如鼠标移入显示区等
     * 从事件队列中拿一个事件，放到event中，如果没有事件，则进入循环中
     * SDL_PeekEvents用于读取事件，在调用该函数之前，必须调用SDL_PumpEvents搜集键盘等事件
     */
    while (!SDL_PeepEvents(event, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT)) {
    
    
        if (!cursor_hidden && av_gettime_relative() - cursor_last_shown > CURSOR_HIDE_DELAY) {
    
    
            SDL_ShowCursor(0);
            cursor_hidden = 1;
        }
        /*
         * remaining_time就是用来进行音视频同步的。
         * 在video_refresh函数中，根据当前帧显示时刻(display time)和实际时刻(actual time)
         * 计算需要sleep的时间，保证帧按时显示
         */
        if (remaining_time > 0.0)   //sleep控制画面输出的时机
            av_usleep((int64_t)(remaining_time * 1000000.0)); // remaining_time <= REFRESH_RATE
        remaining_time = REFRESH_RATE;
        if (is->show_mode != SHOW_MODE_NONE && // 显示模式不等于SHOW_MODE_NONE
            (!is->paused  // 非暂停状态
             || is->force_refresh) // 强制刷新状态
            ) {
    
    
            video_refresh(is, &remaining_time);
        }
        /* 从输入设备中搜集事件，推动这些事件进入事件队列，更新事件队列的状态，
         * 不过它还有一个作用是进行视频子系统的设备状态更新，如果不调用这个函数，
         * 所显示的视频会在大约10秒后丢失色彩。没有调用SDL_PumpEvents，将不会
         * 有任何的输入设备事件进入队列，这种情况下，SDL就无法响应任何的键盘等硬件输入。
        */
        SDL_PumpEvents();
    }
}

SDL_PeepEvent通过SDL_GETEVENT非阻塞查询队列中是否有事件,如果不为0则有事件发生(-1表示发生错误),那么函数就会返回,让event_loop进行处理;否则调用video_refresh进行显示画面,并且通过输出参数remaining_time获取下一轮应当sleep的时间以保证画面稳定输出.

是否调用video_refresh的前置条件为:

1. 显示模式不为SHOW_MODE_NONE(如果只包含audio的画,也可能为其波形图)
2. 当前没有被暂停
3. 当设置了force_refresh(强制刷新),分析一下出现的情况:
   • video_refresh里面帧显示,常规情况.
   • SDL_WINDOWEVENT_EXPOSED,窗口需要重新绘制
   • SDL_MOUSEBUTTONDOWN && SDL_BUTTON_LEFT 鼠标按下并且按左键连续间隔小于0.5s
   • SDLK_f,按f键进行全屏或者恢复原始播放窗口

有可能理解不了这个强制刷新是什么,看一下没有强制刷新的效果图:

可见下面一部分残缺了,因为我们改变了窗口大小,但是我们渲染的页面没有改变,这时我们重新刷新一下可以放渲染页面跟着窗口大小进行改变

1.2.2 video_refresh

static void video_refresh(void *opaque, double *remaining_time)
{
    
    
    VideoState *is = opaque;
    double time;

    Frame *sp, *sp2;

    if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
        check_external_clock_speed(is);

    if (!display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
    
    
        time = av_gettime_relative() / 1000000.0;
        if (is->force_refresh || is->last_vis_time + rdftspeed < time) {
    
    
            video_display(is);
            is->last_vis_time = time;
        }
        *remaining_time = FFMIN(*remaining_time, is->last_vis_time + rdftspeed - time);
    }

    if (is->video_st) {
    
    
    retry:
        if (frame_queue_nb_remaining(&is->pictq) == 0) {
    
    // 帧队列是否为空
            // nothing to do, no picture to display in the queue
            // 什么都不做，队列中没有图像可显示
        } else {
    
     // 重点是音视频同步
            double last_duration, duration, delay;
            Frame *vp, *lastvp;

            /* dequeue the picture */
            // 从队列取出上一个Frame
            lastvp = frame_queue_peek_last(&is->pictq);//读取上一帧
            vp = frame_queue_peek(&is->pictq);  // 读取待显示帧
            // lastvp 上一帧(正在显示的帧)
            // vp 等待显示的帧

            if (vp->serial != is->videoq.serial) {
    
    
                // 如果不是最新的播放序列，则将其出队列，以尽快读取最新序列的帧
                frame_queue_next(&is->pictq);
                goto retry;
            }

            if (lastvp->serial != vp->serial) {
    
    
                // 新的播放序列重置当前时间
                is->frame_timer = av_gettime_relative() / 1000000.0;
            }

            if (is->paused)
            {
    
    
                goto display;
                printf("视频暂停is->paused");
            }
            /* compute nominal last_duration */
            //lastvp上一帧，vp当前帧 ，nextvp下一帧
            //last_duration 计算上一帧应显示的时长
            last_duration = vp_duration(is, lastvp, vp);

            // 经过compute_target_delay方法，计算出待显示帧vp需要等待的时间
            // 如果以video同步，则delay直接等于last_duration。
            // 如果以audio或外部时钟同步，则需要比对主时钟调整待显示帧vp要等待的时间。
            delay = compute_target_delay(last_duration, is); // 上一帧需要维持的时间
            time= av_gettime_relative()/1000000.0;
            // is->frame_timer 实际上就是上一帧lastvp的播放时间,
            // is->frame_timer + delay 是待显示帧vp该播放的时间
            if (time < is->frame_timer + delay) {
    
     //判断是否继续显示上一帧
                // 当前系统时刻还未到达上一帧的结束时刻，那么还应该继续显示上一帧。
                // 计算出最小等待时间
                *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
                goto display;
            }

            // 走到这一步，说明已经到了或过了该显示的时间，待显示帧vp的状态变更为当前要显示的帧

            is->frame_timer += delay;   // 更新当前帧播放的时间
            if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX) {
    
    
                is->frame_timer = time; //如果和系统时间差距太大，就纠正为系统时间
            }
            SDL_LockMutex(is->pictq.mutex);
            if (!isnan(vp->pts))
                update_video_pts(is, vp->pts, vp->pos, vp->serial); // 更新video时钟
            SDL_UnlockMutex(is->pictq.mutex);
            //丢帧逻辑
            if (frame_queue_nb_remaining(&is->pictq) > 1) {
    
    //有nextvp才会检测是否该丢帧
                Frame *nextvp = frame_queue_peek_next(&is->pictq);
                duration = vp_duration(is, vp, nextvp);
                if(!is->step        // 非逐帧模式才检测是否需要丢帧 is->step==1 为逐帧播放
                    && (framedrop>0 ||      // cpu解帧过慢
                        (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) // 非视频同步方式
                    && time > is->frame_timer + duration // 确实落后了一帧数据
                    ) {
    
    
                    printf("%s(%d) dif:%lfs, drop frame\n", __FUNCTION__, __LINE__,
                           (is->frame_timer + duration) - time);
                    is->frame_drops_late++;             // 统计丢帧情况
                    frame_queue_next(&is->pictq);       // 这里实现真正的丢帧
                    //(这里不能直接while丢帧，因为很可能audio clock重新对时了，这样delay值需要重新计算)
                    goto retry; //回到函数开始位置，继续重试
                }
            }

            if (is->subtitle_st) {
    
    
                while (frame_queue_nb_remaining(&is->subpq) > 0) {
    
    
                    sp = frame_queue_peek(&is->subpq);

                    if (frame_queue_nb_remaining(&is->subpq) > 1)
                        sp2 = frame_queue_peek_next(&is->subpq);
                    else
                        sp2 = NULL;

                    if (sp->serial != is->subtitleq.serial
                        || (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
                        || (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
                    {
    
    
                        if (sp->uploaded) {
    
    
                            int i;
                            for (i = 0; i < sp->sub.num_rects; i++) {
    
    
                                AVSubtitleRect *sub_rect = sp->sub.rects[i];
                                uint8_t *pixels;
                                int pitch, j;

                                if (!SDL_LockTexture(is->sub_texture, (SDL_Rect *)sub_rect, (void **)&pixels, &pitch)) {
    
    
                                    for (j = 0; j < sub_rect->h; j++, pixels += pitch)
                                        memset(pixels, 0, sub_rect->w << 2);
                                    SDL_UnlockTexture(is->sub_texture);
                                }
                            }
                        }
                        frame_queue_next(&is->subpq);
                    } else {
    
    
                        break;
                    }
                }
            }

            frame_queue_next(&is->pictq);   // 当前vp帧出队列
            is->force_refresh = 1;          /* 说明需要刷新视频帧 */

            if (is->step && !is->paused)
                stream_toggle_pause(is);    // 逐帧的时候那继续进入暂停状态
        }
    display:
        /* display picture */
        if (!display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
            video_display(is); // 重点是显示
    }
    is->force_refresh = 0;
    if (show_status) {
    
    
        static int64_t last_time;
        int64_t cur_time;
        int aqsize, vqsize, sqsize;
        double av_diff;

        cur_time = av_gettime_relative();
        if (!last_time || (cur_time - last_time) >= 30000) {
    
    
            aqsize = 0;
            vqsize = 0;
            sqsize = 0;
            if (is->audio_st)
                aqsize = is->audioq.size;
            if (is->video_st)
                vqsize = is->videoq.size;
            if (is->subtitle_st)
                sqsize = is->subtitleq.size;
            av_diff = 0;
            if (is->audio_st && is->video_st)
                av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
            else if (is->video_st)
                av_diff = get_master_clock(is) - get_clock(&is->vidclk);
            else if (is->audio_st)
                av_diff = get_master_clock(is) - get_clock(&is->audclk);
            av_log(NULL, AV_LOG_INFO,
                   "%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64"   \r",
                   get_master_clock(is),
                   (is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : "   ")),
                   av_diff,
                   is->frame_drops_early + is->frame_drops_late,
                   aqsize / 1024,
                   vqsize / 1024,
                   sqsize,
                   is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,
                   is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);
            fflush(stdout);
            last_time = cur_time;
        }
    }
}

流程图：

看主流程图：

1. 取出上一帧和待显示的帧
2. 计算上一帧显示的时长,判断当前是否继续上一帧
3. 估算当前帧显示时长,判断是否要丢帧
4. 调用video_display显示

1.2.2.1 计算上一帧显示时长,判断是否还要继续上一帧

首先判断pictq是否为空(调用frame_queue_nb_remaining判断是否还有未显示的帧),如果为空则继续调用video_display显示上一帧

在进一步计算上一帧显示时间之前,需要先判断下一帧vp是否为最新序列,也就是说if(vp->serial!= is->videoq.serial),如果条件成立就是发生过seek等操作,此时应该丢弃lastvp. 故调用frame_queue_next抛弃lastvp后,返回流程开头重试.

接下来 可以计算lastvp显示时长了. 计算代码为:

last_duration = vp_duration(is, lastvp, vp);
delay = compute_target_delay(last_duration, is);//返回当前显示帧要持续播放的时间

本质就是通过上一帧和待显示帧pts来计算的,如果考虑到同步,则还需要考虑当前与主时钟的差距来决定是重复上一帧,还是丢帧,还是正常显示下一帧(待显示帧)

            time= av_gettime_relative()/1000000.0;
            // is->frame_timer 实际上就是上一帧lastvp的播放时间,
            // is->frame_timer + delay 是待显示帧vp该播放的时间
            if (time < is->frame_timer + delay) {
    
     //判断是否继续显示上一帧
                // 当前系统时刻还未到达上一帧的结束时刻，那么还应该继续显示上一帧。
                // 计算出最小等待时间
                *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
                goto display;
            }

frame_timer就是当前帧显示时间,如果当前帧显示时间+delay显示时间大于当前系统时间的话就继续显示上一帧!

1.2.2.2 估算当前帧显示时长,判断是否要丢帧

  is->frame_timer += delay;   // 更新当前帧播放的时间
            if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX) {
    
    
                is->frame_timer = time; //如果和系统时间差距太大，就纠正为系统时间
            }

判断是否需要丢帧

 if (frame_queue_nb_remaining(&is->pictq) > 1) {
    
    //有nextvp才会检测是否该丢帧
                Frame *nextvp = frame_queue_peek_next(&is->pictq);
                duration = vp_duration(is, vp, nextvp);
                if(!is->step        // 非逐帧模式才检测是否需要丢帧 is->step==1 为逐帧播放
                    && (framedrop>0 ||      // cpu解帧过慢
                        (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) // 非视频同步方式
                    && time > is->frame_timer + duration // 确实落后了一帧数据
                    ) {
    
    
                    printf("%s(%d) dif:%lfs, drop frame\n", __FUNCTION__, __LINE__,
                           (is->frame_timer + duration) - time);
                    is->frame_drops_late++;             // 统计丢帧情况
                    frame_queue_next(&is->pictq);       // 这里实现真正的丢帧
                    //(这里不能直接while丢帧，因为很可能audio clock重新对时了，这样delay值需要重新计算)
                    goto retry; //回到函数开始位置，继续重试
                }
            }

通过待显示帧和下一帧来计算当前帧显示时间(前提是得有下一帧)

并且要符合以下条件才会丢帧:

1. 不处于step状态,逐帧状态
2. 启动framedrop模式,也就是cpu过慢时需要丢帧,并且不是以video为同步时钟的情况下
3. 当前时间已经>frame_timer+duration

1.2.2.3 调用video_display进行显示

static void video_display(VideoState *is)
{
    
    
    if (!is->width)
        video_open(is); //如果窗口未显示，则显示窗口

    SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
    SDL_RenderClear(renderer);
    if (is->audio_st && is->show_mode != SHOW_MODE_VIDEO)
        video_audio_display(is);    //图形化显示仅有音轨的文件
    else if (is->video_st)
        video_image_display(is);    //显示一帧视频画面
    SDL_RenderPresent(renderer);
}

static void video_image_display(VideoState *is)
{
    
    
    Frame *vp;
    Frame *sp = NULL;
    SDL_Rect rect;

    // keep_last的作用就出来了,我们是有调用frame_queue_next, 但最近出队列的帧并没有真正销毁
    // 所以这里可以读取出来显示
    vp = frame_queue_peek_last(&is->pictq); //
    if (is->subtitle_st) {
    
    
        if (frame_queue_nb_remaining(&is->subpq) > 0) {
    
    
            sp = frame_queue_peek(&is->subpq);

            if (vp->pts >= sp->pts + ((float) sp->sub.start_display_time / 1000)) {
    
    
                if (!sp->uploaded) {
    
    
                    uint8_t* pixels[4];
                    int pitch[4];
                    int i;
                    if (!sp->width || !sp->height) {
    
    
                        sp->width = vp->width;
                        sp->height = vp->height;
                    }
                    if (realloc_texture(&is->sub_texture, SDL_PIXELFORMAT_ARGB8888, sp->width, sp->height, SDL_BLENDMODE_BLEND, 1) < 0)
                        return;

                    for (i = 0; i < sp->sub.num_rects; i++) {
    
    
                        AVSubtitleRect *sub_rect = sp->sub.rects[i];

                        sub_rect->x = av_clip(sub_rect->x, 0, sp->width );
                        sub_rect->y = av_clip(sub_rect->y, 0, sp->height);
                        sub_rect->w = av_clip(sub_rect->w, 0, sp->width  - sub_rect->x);
                        sub_rect->h = av_clip(sub_rect->h, 0, sp->height - sub_rect->y);

                        is->sub_convert_ctx = sws_getCachedContext(is->sub_convert_ctx,
                                                                   sub_rect->w, sub_rect->h, AV_PIX_FMT_PAL8,
                                                                   sub_rect->w, sub_rect->h, AV_PIX_FMT_BGRA,
                                                                   0, NULL, NULL, NULL);
                        if (!is->sub_convert_ctx) {
    
    
                            av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
                            return;
                        }
                        if (!SDL_LockTexture(is->sub_texture, (SDL_Rect *)sub_rect, (void **)pixels, pitch)) {
    
    
                            sws_scale(is->sub_convert_ctx, (const uint8_t * const *)sub_rect->data, sub_rect->linesize,
                                      0, sub_rect->h, pixels, pitch);
                            SDL_UnlockTexture(is->sub_texture);
                        }
                    }
                    sp->uploaded = 1;
                }
            } else
                sp = NULL;
        }
    }
    //将帧宽高按照sar最大适配到窗口，并通过rect返回视频帧在窗口的显示位置和宽高
    calculate_display_rect(&rect, is->xleft, is->ytop, is->width, is->height,
                           vp->width, vp->height, vp->sar);
    //    rect.x = rect.w /2;   // 测试
    //    rect.w = rect.w /2;   // 缩放实际不是用sws， 缩放是sdl去做的
    if (!vp->uploaded) {
    
    
        // 把yuv数据更新到vid_texture
        if (upload_texture(&is->vid_texture, vp->frame, &is->img_convert_ctx) < 0)
            return;
        vp->uploaded = 1;
        vp->flip_v = vp->frame->linesize[0] < 0;
    }

    set_sdl_yuv_conversion_mode(vp->frame);
    SDL_RenderCopyEx(renderer, is->vid_texture, NULL, &rect, 0, NULL, vp->flip_v ? SDL_FLIP_VERTICAL : 0);
    set_sdl_yuv_conversion_mode(NULL);
    if (sp) {
    
    
#if USE_ONEPASS_SUBTITLE_RENDER
        SDL_RenderCopy(renderer, is->sub_texture, NULL, &rect);
#else
        int i;
        double xratio = (double)rect.w / (double)sp->width;
        double yratio = (double)rect.h / (double)sp->height;
        for (i = 0; i < sp->sub.num_rects; i++) {
    
    
            SDL_Rect *sub_rect = (SDL_Rect*)sp->sub.rects[i];
            SDL_Rect target = {
    
    .x = rect.x + sub_rect->x * xratio,
                               .y = rect.y + sub_rect->y * yratio,
                               .w = sub_rect->w * xratio,
                               .h = sub_rect->h * yratio};
            SDL_RenderCopy(renderer, is->sub_texture, sub_rect, &target);
        }
#endif
    }
}

video_image_display整体不算复杂,每次渲染都会调用calculate_display_rect进行重新计算显示窗口等

最主要的显示是调用upload_texture将AVFrame的图像数据传给sdl的纹理进行渲染:

static int upload_texture(SDL_Texture **tex, AVFrame *frame, struct SwsContext **img_convert_ctx) {
    
    
    int ret = 0;
    Uint32 sdl_pix_fmt;
    SDL_BlendMode sdl_blendmode;
    // 根据frame中的图像格式(FFmpeg像素格式)，获取对应的SDL像素格式和blendmode
    get_sdl_pix_fmt_and_blendmode(frame->format, &sdl_pix_fmt, &sdl_blendmode);
    // 参数tex实际是&is->vid_texture，此处根据得到的SDL像素格式，为&is->vid_texture
    if (realloc_texture(tex, sdl_pix_fmt == SDL_PIXELFORMAT_UNKNOWN ? SDL_PIXELFORMAT_ARGB8888 : sdl_pix_fmt,
                        frame->width, frame->height, sdl_blendmode, 0) < 0)
        return -1;
    //根据sdl_pix_fmt从AVFrame中取数据填充纹理
    switch (sdl_pix_fmt) {
    
    
        // frame格式是SDL不支持的格式，则需要进行图像格式转换，转换为目标格式AV_PIX_FMT_BGRA，
    // 对应SDL_PIXELFORMAT_BGRA32
    case SDL_PIXELFORMAT_UNKNOWN:
        /* This should only happen if we are not using avfilter... */
        *img_convert_ctx = sws_getCachedContext(*img_convert_ctx,
                                                frame->width, frame->height, frame->format,
                                                frame->width, frame->height, AV_PIX_FMT_BGRA,
                                                sws_flags, NULL, NULL, NULL);
        if (*img_convert_ctx != NULL) {
    
    
            uint8_t *pixels[4]; // 之前取Texture的缓存
            int pitch[4];
            if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {
    
    
                sws_scale(*img_convert_ctx, (const uint8_t * const *)frame->data, frame->linesize,
                          0, frame->height, pixels, pitch);
                SDL_UnlockTexture(*tex);
            }
        } else {
    
    
            av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
            ret = -1;
        }
        break;
    // frame格式对应SDL_PIXELFORMAT_IYUV，不用进行图像格式转换，调用SDL_UpdateYUVTexture()更新SDL texture
    case SDL_PIXELFORMAT_IYUV:
        if (frame->linesize[0] > 0 && frame->linesize[1] > 0 && frame->linesize[2] > 0) {
    
    
            ret = SDL_UpdateYUVTexture(*tex, NULL, frame->data[0], frame->linesize[0],
                                       frame->data[1], frame->linesize[1],
                                       frame->data[2], frame->linesize[2]);
        } else if (frame->linesize[0] < 0 && frame->linesize[1] < 0 && frame->linesize[2] < 0) {
    
    
            ret = SDL_UpdateYUVTexture(*tex, NULL, frame->data[0] + frame->linesize[0] * (frame->height                    - 1), -frame->linesize[0],
                                       frame->data[1] + frame->linesize[1] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[1],
                                       frame->data[2] + frame->linesize[2] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[2]);
        } else {
    
    
            av_log(NULL, AV_LOG_ERROR, "Mixed negative and positive linesizes are not supported.\n");
            return -1;
        }
        break;
    // frame格式对应其他SDL像素格式，不用进行图像格式转换，调用SDL_UpdateTexture()更新SDL texture
    default:
        if (frame->linesize[0] < 0) {
    
    
            ret = SDL_UpdateTexture(*tex, NULL, frame->data[0] + frame->linesize[0] * (frame->height - 1), -frame->linesize[0]);
        } else {
    
    
            ret = SDL_UpdateTexture(*tex, NULL, frame->data[0], frame->linesize[0]);
        }
        break;
    }
    return ret;
}

frame中的像素格式是FFmpeg中定义的像素格式,FFmpeg中定义的很多像素格式与SDL中定义的像素格式是同一种格式,只不过是名称不同

根据frame中的像素格式与SDL的像素格式的匹配情况,upload_texture()处理三种类型,对应的是Switch语句的三种分支:

1. 如果frame图像格式对应SDL_PIXELFORMAT_IYUV格式,则不需要图像格式转换,使用SDL_updateYUVBTexture()将数据更新到&is->vid_texture中
2. 如果frame图像格式对应SDL其他支持的格式,也不需要进行图像格式转换,使用SDL_updateTexture()将数据更新到&is->vid_texture中
3. 如果frame图像不被SDL支持的话,则需要进行图像格式转换

根据映射表获取frame对应SDL中的像素格式:

static void get_sdl_pix_fmt_and_blendmode(int format, Uint32 *sdl_pix_fmt, SDL_BlendMode *sdl_blendmode)
{
    
    
    int i;
    *sdl_blendmode = SDL_BLENDMODE_NONE;
    *sdl_pix_fmt = SDL_PIXELFORMAT_UNKNOWN;
    if (format == AV_PIX_FMT_RGB32   ||
        format == AV_PIX_FMT_RGB32_1 ||
        format == AV_PIX_FMT_BGR32   ||
        format == AV_PIX_FMT_BGR32_1)
        *sdl_blendmode = SDL_BLENDMODE_BLEND;
    for (i = 0; i < FF_ARRAY_ELEMS(sdl_texture_format_map) - 1; i++) {
    
    
        if (format == sdl_texture_format_map[i].format) {
    
    
            *sdl_pix_fmt = sdl_texture_format_map[i].texture_fmt;
            return;
        }
    }
}

映射表:

    static const struct TextureFormatEntry {
    
    
    enum AVPixelFormat format;
    int texture_fmt;
    }
sdl_texture_format_map[] = {
    
      // FFmpeg PIX_FMT to SDL_PIX的映射关系
        {
    
     AV_PIX_FMT_RGB8,           SDL_PIXELFORMAT_RGB332 },
        {
    
     AV_PIX_FMT_RGB444,         SDL_PIXELFORMAT_RGB444 },
        {
    
     AV_PIX_FMT_RGB555,         SDL_PIXELFORMAT_RGB555 },
        {
    
     AV_PIX_FMT_BGR555,         SDL_PIXELFORMAT_BGR555 },
        {
    
     AV_PIX_FMT_RGB565,         SDL_PIXELFORMAT_RGB565 },
        {
    
     AV_PIX_FMT_BGR565,         SDL_PIXELFORMAT_BGR565 },
        {
    
     AV_PIX_FMT_RGB24,          SDL_PIXELFORMAT_RGB24 },
        {
    
     AV_PIX_FMT_BGR24,          SDL_PIXELFORMAT_BGR24 },
        {
    
     AV_PIX_FMT_0RGB32,         SDL_PIXELFORMAT_RGB888 },
        {
    
     AV_PIX_FMT_0BGR32,         SDL_PIXELFORMAT_BGR888 },
        {
    
     AV_PIX_FMT_NE(RGB0, 0BGR), SDL_PIXELFORMAT_RGBX8888 },
        {
    
     AV_PIX_FMT_NE(BGR0, 0RGB), SDL_PIXELFORMAT_BGRX8888 },
        {
    
     AV_PIX_FMT_RGB32,          SDL_PIXELFORMAT_ARGB8888 },
        {
    
     AV_PIX_FMT_RGB32_1,        SDL_PIXELFORMAT_RGBA8888 },
        {
    
     AV_PIX_FMT_BGR32,          SDL_PIXELFORMAT_ABGR8888 },
        {
    
     AV_PIX_FMT_BGR32_1,        SDL_PIXELFORMAT_BGRA8888 },
        {
    
     AV_PIX_FMT_YUV420P,        SDL_PIXELFORMAT_IYUV },
        {
    
     AV_PIX_FMT_YUYV422,        SDL_PIXELFORMAT_YUY2 },
        {
    
     AV_PIX_FMT_UYVY422,        SDL_PIXELFORMAT_UYVY },
        {
    
     AV_PIX_FMT_NONE,           SDL_PIXELFORMAT_UNKNOWN },
        };

可以看到，除了最后⼀项，其他格式的图像送给SDL是可以直接显示的，不必进行图像转换。

1.2.2.4 realloc_texture()重新分配vid_texture

static int realloc_texture(SDL_Texture **texture, Uint32 new_format, int new_width, int new_height,
                           SDL_BlendMode blendmode, int init_texture)
{
    
    
    Uint32 format;
    int access, w, h;
    if (!*texture || SDL_QueryTexture(*texture, &format, &access, &w, &h) < 0 || new_width != w || new_height != h || new_format != format) {
    
    
        void *pixels;
        int pitch;
        if (*texture)
            SDL_DestroyTexture(*texture);
        if (!(*texture = SDL_CreateTexture(renderer, new_format, SDL_TEXTUREACCESS_STREAMING, new_width, new_height)))
            return -1;
        if (SDL_SetTextureBlendMode(*texture, blendmode) < 0)
            return -1;
        if (init_texture) {
    
    
            if (SDL_LockTexture(*texture, NULL, &pixels, &pitch) < 0)
                return -1;
            memset(pixels, 0, pitch * new_height);
            SDL_UnlockTexture(*texture);
        }
        av_log(NULL, AV_LOG_VERBOSE, "Created %dx%d texture with %s.\n", new_width, new_height, SDL_GetPixelFormatName(new_format));
    }
    return 0;
}

什么情况下需要realloc_texture?

1. 用于显示的texture没有分配
2. SDL_QueryTexture无效
3. 目前texture的width,height,format和新药显示的Frame不一致

综上所述:窗口大小变化不足以让realloc_texture重新SDL_CreateTextue

1.2.2.5 sws_getCachedContext

struct SwsContext *sws_getCachedContext(struct SwsContext *context,
                                        int srcW, int srcH, enum AVPixelFormat srcFormat,
                                        int dstW, int dstH, enum AVPixelFormat dstFormat,
                                        int flags, SwsFilter *srcFilter,
                                        SwsFilter *dstFilter, const double *param);

创建一个图像转换的上下文

这里需要说明的flags这个参数,这个参数是选择转换算法的,有很多转换算法,在libswscale/swscale.h文件中

1.2.2.6 sws_scale 图像转换

int sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[],
              const int srcStride[], int srcSliceY, int srcSliceH,
              uint8_t *const dst[], const int dstStride[]);

            if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {
    
    
                sws_scale(*img_convert_ctx, (const uint8_t * const *)frame->data, frame->linesize,
                          0, frame->height, pixels, pitch);
                SDL_UnlockTexture(*tex);
            }

对于flags算法测试推荐文章:

(66条消息) ffmpeg中的sws_scale算法性能测试_ffmpeg算法_雷霄骅的博客-CSDN博客