GStreamer字幕解码模块实现

  GStreamer是一个灵活、快速的特别适合作为音视频开发的框架。其Pipeline和插件等设计思想使其特别适合快速组装出一个复杂的多媒体应用。

  然而，近一年用GStreamer以来发现其对字幕的支持特别有限，表现在一下方面，

既有情况：

    1. 字幕的解码器缺乏，如缺少pgs、dvd、dvb、xsub等图形字幕的独立解码模块支持

    2. 其基于sink端字幕subtitle overlay于video之上的机制，难以实现SOC上的零拷贝显示输出

设计思想：

    因此计划开发GStreamer上以字幕流作为输入，RGBA格式作为输出的字幕解码模块。实际应用端可以获取解码后RGBA的Bitmap贴图灵活实现各种应用场景。

    选取FFmpeg作为字幕解码器，解码后将AVSubtitle数据填充至GstBuffer中，并标记format、宽高、时间戳。

实现：

    1. 注册组件

        遍历FFmpeg中的图形字幕解码器创建对于组件

gboolean
gst_ffmpegsubdec_register (GstPlugin * plugin)
{
  GTypeInfo typeinfo = {
    sizeof (GstFFMpegSubDecClass),
    (GBaseInitFunc) gst_ffmpeg_sub_dec_base_init,
    NULL,
    (GClassInitFunc) gst_ffmpeg_sub_dec_class_init,
    NULL,
    NULL,
    sizeof (GstFFMpegSubDec),
    0,
    (GInstanceInitFunc) gst_ffmpeg_sub_dec_init,
  };
  GType type;
  AVCodec *in_plugin;
  gint rank;

  in_plugin = av_codec_next(NULL);

  GST_LOG ("Registering decoders");

  while (in_plugin) {
    gchar *type_name;

    /* only decoders */
    if (!av_codec_is_decoder (in_plugin)
        || in_plugin->type != AVMEDIA_TYPE_SUBTITLE) {
      goto next;
    }

    switch (in_plugin->id) {
    case AV_CODEC_ID_HDMV_PGS_SUBTITLE:
    case AV_CODEC_ID_DVD_SUBTITLE:
    case AV_CODEC_ID_DVB_SUBTITLE:
    case AV_CODEC_ID_XSUB:
      break;
    default:
      goto next;
    }

    GST_DEBUG ("Trying plugin %s [%s]", in_plugin->name, in_plugin->long_name);

    /* construct the type */
    type_name = g_strdup_printf ("avdec_%s", in_plugin->name);
    g_strdelimit (type_name, ".,|-<> ", '_');

    type = g_type_from_name (type_name);

    if (!type) {
      /* create the gtype now */
      type =
          g_type_register_static (GST_TYPE_FFMPEG_SUB_DEC, type_name, &typeinfo,
          0);
      g_type_set_qdata (type, GST_FFDEC_PARAMS_QDATA, (gpointer) in_plugin);
    }

    switch(in_plugin->id){
    case AV_CODEC_ID_HDMV_PGS_SUBTITLE:
    case AV_CODEC_ID_DVD_SUBTITLE:
      rank = GST_RANK_PRIMARY;
      break;
    default:
      rank = GST_RANK_SECONDARY;
    };

    if (!gst_element_register (plugin, type_name, rank, type)) {
      g_warning ("Failed to register %s", type_name);
      g_free (type_name);
      return FALSE;
    }

    g_free (type_name);

  next:
    in_plugin = av_codec_next (in_plugin);
  }

  GST_LOG ("Finished Registering decoders");

  return TRUE;
}

    2. 初始化解码上下文

        在组件构造函数中，初始化和开启FFmpeg解码上下文和解码器

static void gst_ffmpeg_sub_dec_init(GstFFMpegSubDec * ffmpegdec)
{
  GstFFMpegSubDecClass *klass =
      (GstFFMpegSubDecClass *) G_OBJECT_GET_CLASS (ffmpegdec);

  ffmpegdec->opened = FALSE;

  if (!klass->in_plugin)
    return;

  ffmpegdec->sinkpad = gst_pad_new_from_template (klass->sinktempl, "sink");
  gst_pad_set_chain_function (ffmpegdec->sinkpad,
      GST_DEBUG_FUNCPTR (gst_ffmpegsubdec_chain));
  gst_pad_set_event_function (ffmpegdec->sinkpad,
      GST_DEBUG_FUNCPTR (gst_ffmpegsubdec_sink_event));
  gst_element_add_pad (GST_ELEMENT (ffmpegdec), ffmpegdec->sinkpad);

  ffmpegdec->srcpad = gst_pad_new_from_template (klass->srctempl, "src");
  gst_pad_set_event_function (ffmpegdec->srcpad,
      GST_DEBUG_FUNCPTR (gst_ffmpegdec_src_event));
  gst_element_add_pad (GST_ELEMENT (ffmpegdec), ffmpegdec->srcpad);

  /* some ffmpeg data */
  ffmpegdec->codec = avcodec_find_decoder(klass->in_plugin->id);
  if (ffmpegdec->codec) {
    GST_DEBUG_OBJECT(ffmpegdec, "try open codec %d name %s\n", ffmpegdec->codec->id, ffmpegdec->codec->name);
    ffmpegdec->context = avcodec_alloc_context3 (ffmpegdec->codec);
    ffmpegdec->context->opaque = ffmpegdec;

    ffmpegdec->frame = av_mallocz(sizeof(AVSubtitle));

    if (gst_ffmpeg_avcodec_open (ffmpegdec->context, ffmpegdec->codec) < 0)
      goto could_not_open;

    ffmpegdec->opened = TRUE;

    GST_DEBUG_OBJECT (ffmpegdec, "Opened libav codec %s, id %d",
        ffmpegdec->codec->name, ffmpegdec->codec->id);
  }

  return;

could_not_open:

  gst_ffmpeg_avcodec_close (ffmpegdec->context);
  GST_DEBUG_OBJECT (ffmpegdec, "avdec_%s: Failed to open libav codec",
      klass->in_plugin->name);
};

    3. 调用解码接口

      实现chain接口，在函数中调用avcodec_decode_subtitle2调用FFmpeg解析字幕

static GstFlowReturn
gst_ffmpegsubdec_chain (GstPad * pad, GstObject * parent, GstBuffer * inbuf)
{
  GstFlowReturn ret = GST_FLOW_OK;
  GstFFMpegSubDec *ffmpegdec;
  GstMapInfo map;
  guint8 *bdata;
  glong bsize = 0;
  guint8 retry = 10;

  ffmpegdec = GST_FFMPEG_SUB_DEC (parent);

  GST_DEBUG_OBJECT (ffmpegdec, "Have buffer of size %" G_GSIZE_FORMAT ", ts %"
      GST_TIME_FORMAT ", dur %" G_GINT64_FORMAT, gst_buffer_get_size (inbuf),
      GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (inbuf)), GST_BUFFER_DURATION (inbuf));

  inbuf = gst_buffer_ref (inbuf);
  gst_buffer_map (inbuf, &map, GST_MAP_READ);

  bdata = map.data;
  bsize = map.size;

  do {
    AVPacket packet = {0};
    gint have_data = 0;    
    gint len = -1;

    if (ffmpegdec->context->codec_id == AV_CODEC_ID_DVB_SUBTITLE) {
      if (bsize >= 3 && bdata[0] == 0x20 && bdata[1] == 0x00) {
        bdata += 2;
        bsize -= 2;
      }
    }

    gst_avpacket_init (&packet, bdata, bsize);

    len = avcodec_decode_subtitle2(ffmpegdec->context, ffmpegdec->frame,
                               &have_data,
                               &packet);

    if (len >= 0) {
      bdata += len;
      bsize -= len;
    }

    if (have_data) {
      if (ffmpegdec->frame->format == 0 /* graphics */) {
        switch(ffmpegdec->context->codec_id) {
        case AV_CODEC_ID_HDMV_PGS_SUBTITLE:
        case AV_CODEC_ID_DVB_SUBTITLE:
        case AV_CODEC_ID_DVD_SUBTITLE:
        case AV_CODEC_ID_XSUB:
          gst_ffmpegsubdec_picture(ffmpegdec, ffmpegdec->frame, 
            GST_BUFFER_TIMESTAMP (inbuf), GST_BUFFER_DURATION (inbuf));
          break;
        default:
          break;
        }
      }
      avsubtitle_free(ffmpegdec->frame);
    }
  } 
  while (bsize > 0 && retry--);

  gst_buffer_unmap(inbuf, &map);
  gst_buffer_unref(inbuf);

  return ret;
}

    4. 绘制RGBA Bitmap

      解析avsubtitle结构体，创建GstBuffer填充RGBA数据添加format、宽高和时间戳，并push到src pad

static void gst_ffmpegsubdec_picture(GstFFMpegSubDec *ffmpegdec, AVSubtitle* sub, GstClockTime ts, GstClockTime dur)
{
  gint i, j, w, h, size;
  guint8 **data;
  guint8 *ptr;
  GstBuffer *buffer;

  if (sub && sub->num_rects) {
    for (i = 0; i < sub->num_rects; i++) {
      w = sub->rects[i]->w;
      h = sub->rects[i]->h;

      size = w*h*4;
      data = sub->rects[i]->pict.data;
      ptr = g_malloc(size);
      for (j = 0; j < w * h; j++) {
        gint index = data[0][j];
        guint8 r = data[1][index*4];
        guint8 g = data[1][index*4 + 1];
        guint8 b = data[1][index*4 + 2];
        guint8 a = data[1][index*4 + 3];

        ptr[4*j + 0] = a*r/255;
        ptr[4*j + 1] = a*g/255;
        ptr[4*j + 2] = a*b/255;
        ptr[4*j + 3] = a*a/255;
      }
      buffer = gst_buffer_new_allocate (NULL, size, NULL);
      gst_buffer_fill(buffer, 0, ptr, size);
      gst_buffer_add_video_meta (buffer, GST_VIDEO_FRAME_FLAG_NONE,
        GST_VIDEO_FORMAT_RGBA, w, h);
      GST_BUFFER_PTS(buffer) = ts;
      GST_BUFFER_DURATION(buffer) = dur;

      GST_DEBUG_OBJECT(ffmpegdec, "Have picture w:%d, h:%d, ts %"
        GST_TIME_FORMAT ", dur %" G_GINT64_FORMAT, w, h, GST_TIME_ARGS (ts), GST_BUFFER_DURATION (buffer));

      gst_pad_push(ffmpegdec->srcpad, buffer);
      g_free(ptr);
    }
  }else {
     buffer = gst_buffer_new_allocate (NULL, 0, NULL);
     gst_buffer_add_video_meta (buffer, GST_VIDEO_FRAME_FLAG_NONE,
        GST_VIDEO_FORMAT_RGBA, 0, 0);
     GST_BUFFER_PTS(buffer) = ts;
     GST_BUFFER_DURATION(buffer) = dur;
     gst_pad_push(ffmpegdec->srcpad, buffer);

     GST_DEBUG_OBJECT(ffmpegdec, "Have empty picture ts %"
        GST_TIME_FORMAT ", dur %" G_GINT64_FORMAT, GST_TIME_ARGS (ts), GST_BUFFER_DURATION (buffer));
  }
}

   5. 应用

    用户和客户在使用过程中，设置playbin的字幕的显示端text-sink为appsink，应用从appsink中获取到字幕buffer后将其贴图到UI的指定区域实现字幕绘制。

   具体代码以上传供参考：https://github.com/JamesLinEngineer/gst-libav-rk/blob/3.3/ext/libav/gstavsubdec.c