Android提供给应用编解码的接口为MediaCodec。我们这里从NuPlayerDecoder开始分析，一是为了衔接之前将的MediaPlayer-NuPlayer流程，二是可以从中参考到MediaCodec是怎么用的，然后去分析内部流程会更好。
之前说到NuPlayer.cpp创建Decoder流程（Video部分）：instantiateDecoder
1965 sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
1966 ++mVideoDecoderGeneration;
1967 notify->setInt32("generation", mVideoDecoderGeneration);
1968
1969 *decoder = new Decoder(
1970 notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
1971 mVideoDecoderError = false;NuPlayer::Decoder是继承于DecoderBase的，不过DecoderBase主要就是用于处理一些基本的流程（setParameter、start、stop等）。如果Decoder重写了就更不需要看了。Decoder构造函数中主要是创建了一个looper，每一个Decoder都需要有一个looper，因为MediaCodec是阻塞式的调用，但是NuPlayer需要的是异步操作:
Decoder::Decoder
94 mCodecLooper = new ALooper;
95 mCodecLooper->setName("NPDecoder-CL");
96 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
97 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;再回来init：NuPlayer.cpp::instantiateDecoder
2032 (*decoder)->init();
2042 (*decoder)->configure(format);Init()其实主要就是注册handler，主要看下configure，DecoderBase的未重写函数，最终发送kWhatConfigure，调用NuPlayer::Decoder::onConfigure：
293 AString mime;
294 CHECK(format->findString("mime", &mime));
295
296 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
297 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
298
299 mComponentName = mime;
300 mComponentName.append(" decoder");
301 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
302
303 mCodec = MediaCodec::CreateByType(
304 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
305 int32_t secure = 0;
306 if (format->findInt32("secure", &secure) && secure != 0) {
307 if (mCodec != NULL) {
308 mCodec->getName(&mComponentName);
309 mComponentName.append(".secure");
310 mCodec->release();
311 ALOGI("[%s] creating", mComponentName.c_str());
312 mCodec = MediaCodec::CreateByComponentName(
313 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
314 }
315 }可以看到MediaCodec创建有两种方式
- CreateByType
- CreateByComponentName
CreateByType是通过mime创建，而CreateByComponentName是通过组件名创建。当format的secure是1的时候会获取之前mime创建mCodec的组件名，并释放，然后在组件名后拼接secure，然后再通过CreateByComponentName创建codec。Secure类型比较典型的就是youtube视频。（组件名一般为OMX.qcom.video.decoder.avc这种，对应media_codec.xml中）。
再看一下后面几个重要的调用：
NuPlayer::Decoder::onConfigure：
360 err = mCodec->configure(
361 format, mSurface, crypto, 0 /* flags */);
//新建一个AMessage传给mediacodec，当mediacodec做好一些处理（如编码一帧）就把buffer index、flag、size等传入msg，然后post回来，相当于一个callback。
388 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
389 mCodec->setCallback(reply);
390
391 err = mCodec->start();然后看下MediaCodec对应的处理吧：
CreateByType
439 sp<MediaCodec> MediaCodec::CreateByType(
440 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
441 uid_t uid) {
442 Vector<AString> matchingCodecs;
443
444 MediaCodecList::findMatchingCodecs(
445 mime.c_str(),
446 encoder,
447 0,
448 &matchingCodecs);
449
450 if (err != NULL) {
451 *err = NAME_NOT_FOUND;
452 }
453 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
454 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
455 AString componentName = matchingCodecs[i];
456 status_t ret = codec->init(componentName);
457 if (err != NULL) {
458 *err = ret;
459 }
460 if (ret == OK) {
461 return codec;
462 }
463 ALOGD("Allocating component '%s' failed (%d), try next one.",
464 componentName.c_str(), ret);
465 }
466 return NULL;
467 }CreateByComponentName
470 sp<MediaCodec> MediaCodec::CreateByComponentName(
471 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
472 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
473
474 const status_t ret = codec->init(name);
475 if (err != NULL) {
476 *err = ret;
477 }
478 return ret == OK ? codec : NULL; // NULL deallocates codec.
479 }findMatchingCodecs其实就是从所有的mediaCodec数组中选取匹配mime，mediaCodec数组是加载MediaCodecList的时候从media_codecs.xml中读取到的，具体流程放在Extend中跟踪。
CreateByComponentName和CreateByType最终都是new了MediaCodec，调用init传入组件名
887 status_t MediaCodec::init(const AString &name) {
906 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
...
911 for (const AString &codecName : {
name, tmp }) {
912 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
...
916 mCodecInfo = mcl->getCodecInfo(codecIdx);
917 Vector<AString> mediaTypes;
918 mCodecInfo->getSupportedMediaTypes(&mediaTypes);
...
//根据组件名的开头选择CodecBase，如omx.则为ACodec，C2则为CCodec。
931 mCodec = GetCodecBase(name, mCodecInfo->getOwnerName());
//传给ACodec两个callback，参数都为AMessage，what为kWhatCodecNotify，用于ACodec消息回调。
//CodecCallback主要作用为状态回调，BufferChannel主要作用为buffer状态的回调。
951 mCodec->setCallback(
952 std::unique_ptr<CodecBase::CodecCallback>(
953 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
954 mBufferChannel = mCodec->getBufferChannel();
955 mBufferChannel->setCallback(
956 std::unique_ptr<CodecBase::BufferCallback>(
957 new BufferCallback(new AMessage(kWhatCodecNotify,
this))));
...
959 sp<AMessage> msg = new AMessage(kWhatInit, this);
986 err = PostAndAwaitResponse(msg, &response);看一下kWhatInit的消息处理：
2355 case kWhatInit:
2377 mCodec->initiateAllocateComponent(format);上面提到了ACodec/CCodec/MediaFilter都是GetCodecBase通过组件名来创建的。这里拿ACodec当例子，如下为ACodec的构造函数，可以看到ACodec创建了九种状态机，ACodec在不同阶段会通过changeState切换不同的状态，用于管理底层各个Component的各种状态，理解状态机也能帮忙我们理解ACodec工作流程，这里可以看到初始化后状态为mUninitializedState。具体每一个状态机的相互切换留在写数据流的时候再写。
581
582 mUninitializedState = new UninitializedState(this);
583 mLoadedState = new LoadedState(this);
584 mLoadedToIdleState = new LoadedToIdleState(this);
585 mIdleToExecutingState = new IdleToExecutingState(this);
586 mExecutingState = new ExecutingState(this);
587
588 mOutputPortSettingsChangedState =
589 new OutputPortSettingsChangedState(this);
590
591 mExecutingToIdleState = new ExecutingToIdleState(this);
592 mIdleToLoadedState = new IdleToLoadedState(this);
593 mFlushingState = new FlushingState(this);
594
595 mPortEOS[kPortIndexInput] = mPortEOS[kPortIndexOutput] = false;
596 mInputEOSResult = OK;
597
598 mPortMode[kPortIndexInput] = IOMX::kPortModePresetByteBuffer;
599 mPortMode[kPortIndexOutput] = IOMX::kPortModePresetByteBuffer;
600
601 memset(&mLastNativeWindowCrop, 0, sizeof(mLastNativeWindowCrop));
602
603 changeState(mUninitializedState);之后调用initiateAllocateComponent。看下代码：
634 void ACodec::initiateAllocateComponent(const sp<AMessage> &msg) {
635 msg->setWhat(kWhatAllocateComponent);
636 msg->setTarget(this);
637 msg->post();
6653 case ACodec::kWhatAllocateComponent:
6654 {
6655 onAllocateComponent(msg);
6656 handled = true;
6657 break;
6705 bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg){
6710 sp<AMessage> notify = new AMessage(kWhatOMXMessageList, mCodec);
6724 CHECK(msg->findString("componentName", &componentName));
6726 sp<CodecObserver> observer = new CodecObserver(notify);
6731 OMXClient client;
6732 if (client.connect(owner.c_str()) != OK) {
6733 mCodec->signalError(OMX_ErrorUndefined, NO_INIT);
6734 return false;
6735 }
6736 omx = client.interface();
//提高线程调度优先级
6739 int prevPriority = androidGetThreadPriority(tid);
6740 androidSetThreadPriority(tid, ANDROID_PRIORITY_FOREGROUND);
6741 err = omx->allocateNode(componentName.c_str(), observer, &omxNode);
6742 androidSetThreadPriority(tid, prevPriority);
//通过CodecCallback调用MediaCodec的onComponentAllocated来通知MediaCodec当前状态
6769 mCodec->mOMX = omx;
6770 mCodec->mOMXNode = omxNode;
6771 mCodec->mCallback->onComponentAllocated(mCodec->mComponentName.c_str());接下来主要看下OMX的allocateNode，OMX对象是通过client.interface()获取到的，首先看下OMX是怎么起来的，如下代码可以看到OMX服务是跑在MediaCodec进程中，很标准的HIDL实现。hal接口定义在hardware/hardware/interfaces/media/omx/1.0中。
main_codecservice.cpp
51 using namespace ::android::hardware::media::omx::V1_0;
52 sp<IOmx> omx = new implementation::Omx();
55 } else if (omx->registerAsService() != OK) {
60 sp<IOmxStore> omxStore = new implementation::OmxStore(omx);
63 } else if (omxStore->registerAsService() != OK) {
OmxStore和Omx构造函数除了加载mediacodec xml，主要还new了OMXMaster，OMXMaster比较重要的就是添加了各种Plugin，Plugin如其名，就是加载OMX具体组件的插件。一般android原生都是加载soft plugin，各大平台厂商在实现自己硬编解码的时候也会添加自己的plugin。
32 OMXMaster::OMXMaster() {
...
53 addVendorPlugin();
54 addPlatformPlugin();
55 }
61 void OMXMaster::addVendorPlugin() {
62 addPlugin("libstagefrighthw.so");
63 }
64
65 void OMXMaster::addPlatformPlugin() {
66 addPlugin("libstagefright_softomx_plugin.so");
67 }
69 void OMXMaster::addPlugin(const char *libname) {
70 void *libHandle = android_load_sphal_library(libname, RTLD_NOW);
71
72 if (libHandle == NULL) {
73 return;
74 }
75
76 typedef OMXPluginBase *(*CreateOMXPluginFunc)();
77 CreateOMXPluginFunc createOMXPlugin =
78 (CreateOMXPluginFunc)dlsym(
79 libHandle, "createOMXPlugin");
80 if (!createOMXPlugin)
81 createOMXPlugin = (CreateOMXPluginFunc)dlsym(
82 libHandle, "_ZN7android15createOMXPluginEv");
83
84 OMXPluginBase *plugin = nullptr;
85 if (createOMXPlugin) {
86 plugin = (*createOMXPlugin)();
87 }
88
89 if (plugin) {
90 mPlugins.push_back({
plugin, libHandle });
91 addPlugin(plugin);
92 } else {
93 android_unload_sphal_library(libHandle);
94 }
95 }
97 void OMXMaster::addPlugin(OMXPluginBase *plugin) {
98 Mutex::Autolock autoLock(mLock);
104 while ((err = plugin->enumerateComponents(
105 name, sizeof(name), index++)) == OMX_ErrorNone) {
108 if (mPluginByComponentName.indexOfKey(name8) >= 0) {
112 continue;
113 }
115 mPluginByComponentName.add(name8, plugin);
116 }回到ACodec，之前调用了omx->allocateNode(componentName.c_str(), observer, &omxNode);
84 Return<void> Omx::allocateNode(
85 const hidl_string& name,
86 const sp<IOmxObserver>& observer,
87 allocateNode_cb _hidl_cb) {
//OMXNodeInstance相当于组件的句柄，对象需要传回给ACodec，
//component就是通过Instance回调一些Codec函数，传回一些状态以及buffer信息。
100 instance = new OMXNodeInstance(
101 this, new LWOmxObserver(observer), name.c_str());
//调用OmxMaster::makeComponentInstance
104 OMX_ERRORTYPE err = mMaster->makeComponentInstance(
105 name.c_str(), &OMXNodeInstance::kCallbacks,
106 instance.get(), &handle);
116 instance->setHandle(handle);
//传给ACodec instance
144 _hidl_cb(toStatus(OK), new TWOmxNode(instance));
}
146 OMX_ERRORTYPE OMXMaster::makeComponentInstance(
147 const char *name,
148 const OMX_CALLBACKTYPE *callbacks,
149 OMX_PTR appData,
150 OMX_COMPONENTTYPE **component) {
//mPluginByComponentName是之前加载plugin存入的
156 ssize_t index = mPluginByComponentName.indexOfKey(String8(name));
162 OMXPluginBase *plugin = mPluginByComponentName.valueAt(index);
//调用plugin中makeComponentInstance
163 OMX_ERRORTYPE err =
164 plugin->makeComponentInstance(name, callbacks, appData, component);
170 mPluginByInstance.add(*component, plugin);很多平台厂商的plugin文件的实际函数逻辑都在so文件中，无法查看，这里以Soft Plugin作为例子：
87 OMX_ERRORTYPE SoftOMXPlugin::makeComponentInstance(
88 const char *name,
89 const OMX_CALLBACKTYPE *callbacks,
90 OMX_PTR appData,
91 OMX_COMPONENTTYPE **component) {
94 for (size_t i = 0; i < kNumComponents; ++i) {
95 if (strcmp(name, kComponents[i].mName)) {
96 continue;
97 }
//libstagefright_soft_拼接上组件名
99 AString libName = "libstagefright_soft_";
100 libName.append(kComponents[i].mLibNameSuffix);
101 libName.append(".so");
//dlopen打开软编解码器so
117 void *libHandle = dlopen(libName.c_str(), RTLD_NOW|RTLD_NODELETE);
129 CreateSoftOMXComponentFunc createSoftOMXComponent =
130 (CreateSoftOMXComponentFunc)dlsym(
131 libHandle,
132 "_Z22createSoftOMXComponentPKcPK16OMX_CALLBACKTYPE"
133 "PvPP17OMX_COMPONENTTYPE");
142 sp<SoftOMXComponent> codec =
143 (*createSoftOMXComponent)(name, callbacks, appData, component);
160 codec->incStrong(this);
161 codec->setLibHandle(libHandle);以libstagefright_soft_m4vh263dec.so为例
415 android::SoftOMXComponent *createSoftOMXComponent(
416 const char *name, const OMX_CALLBACKTYPE *callbacks,
417 OMX_PTR appData, OMX_COMPONENTTYPE **component) {
418 using namespace android;
419 if (!strcmp(name, "OMX.google.h263.decoder")) {
420 return new android::SoftMPEG4(
421 name, "video_decoder.h263", OMX_VIDEO_CodingH263,
422 kH263ProfileLevels, ARRAY_SIZE(kH263ProfileLevels),
423 callbacks, appData, component);
424 } else if (!strcmp(name, "OMX.google.mpeg4.decoder")) {
425 return new android::SoftMPEG4(
426 name, "video_decoder.mpeg4", OMX_VIDEO_CodingMPEG4,
427 kM4VProfileLevels, ARRAY_SIZE(kM4VProfileLevels),
428 callbacks, appData, component);
429 } else {
430 CHECK(!"Unknown component");
431 }
432 return NULL;
433 }最终根据传入组件名创建相应解码器SoftMPEG4,看了下SoftMPEG4继承于SoftVideoDecoderOMXComponent，而SoftVideoDecoderOMXComponent又继承于SimpleSoftOMXComponent，所以emptyThisBuffer、fillThisBuffer等omx基本方法，都在SimpleSoftOMXComponent中定义，而SoftMPEG4没有重写。至此omx组件创建完成，后续章节通过OMX各个状态的切换以及ACodec对应状态机的切换来继续分析数据流走向，同时会整理omx框架每个部分的关连。
至于MediaFilter(滤波器)、Codec2，大体流程跟ACodec都差不多，不过多分析，Codec2在Q版本及之前基本都是用于管理软编解码器，R版本应该会兼容硬编解码器，大概率会完全替换OMX框架。补一张整体流程图：
Extend
MediaCodecList加载xml
void MediaCodecList::findMatchingCodecs(
346 const char *mime, bool encoder, uint32_t flags,
347 Vector<AString> *matches) {
348 matches->clear();
349
350 const sp<IMediaCodecList> list = getInstance();
351 if (list == nullptr) {
352 return;
353 }
354
355 size_t index = 0;
356 for (;;) {
357 ssize_t matchIndex =
358 list->findCodecByType(mime, encoder, index);
359
360 if (matchIndex < 0) {
361 break;
362 }
363
364 index = matchIndex + 1;
365
366 const sp<MediaCodecInfo> info = list->getCodecInfo(matchIndex);
367 CHECK(info != nullptr);
368 AString componentName = info->getCodecName();
369
370 if ((flags & kHardwareCodecsOnly) && isSoftwareCodec(componentName)) {
371 ALOGV("skipping SW codec '%s'", componentName.c_str());
372 } else {
373 matches->push(componentName);
374 ALOGV("matching '%s'", componentName.c_str());
375 }
376 }
377 //所以更改软硬编解码优先可以在这修改
378 if (flags & kPreferSoftwareCodecs ||
379 property_get_bool("debug.stagefright.swcodec", false)) {
380 matches->sort(compareSoftwareCodecsFirst);
381 }
382 }
383 getInstance的流程跟起来最后都是到了getLocalInstance:
143 sp<IMediaCodecList> MediaCodecList::getLocalInstance() {
MediaCodecList *codecList = new MediaCodecList(GetBuilders());主要就是new了MediaCodecList，关键是参数GetBuilders（），先看omx的话，MediaCodecList中的builder就是OmxInfoBuilder 。
80 OmxInfoBuilder sOmxInfoBuilder{
true /* allowSurfaceEncoders */};
95 std::vector<MediaCodecListBuilderBase *> GetBuilders() {
96 std::vector<MediaCodecListBuilderBase *> builders;
97 // if plugin provides the input surface, we cannot use OMX video encoders.
98 // In this case, rely on plugin to provide list of OMX codecs that are usable.
99 sp<PersistentSurface> surfaceTest =
100 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
101 if (surfaceTest == nullptr) {
102 ALOGD("Allowing all OMX codecs");
103 builders.push_back(&sOmxInfoBuilder);
104 } else {
105 ALOGD("Allowing only non-surface-encoder OMX codecs");
106 builders.push_back(&sOmxNoSurfaceEncoderInfoBuilder);
107 }
108 builders.push_back(GetCodec2InfoBuilder());
109 return builders;
110 }看一下MediaCodecList的构造函数：
203 MediaCodecList::MediaCodecList(std::vector<MediaCodecListBuilderBase*> builders) {
204 mGlobalSettings = new AMessage();
205 mCodecInfos.clear();
206 MediaCodecListWriter writer;
207 for (MediaCodecListBuilderBase *builder : builders) {
208 if (builder == nullptr) {
209 ALOGD("ignored a null builder");
210 continue;
211 }
212 mInitCheck = builder->buildMediaCodecList(&writer);
213 if (mInitCheck != OK) {
214 break;
215 }
216 }
217 writer.writeGlobalSettings(mGlobalSettings);
218 writer.writeCodecInfos(&mCodecInfos);
219 std::stable_sort(
220 mCodecInfos.begin(),
221 mCodecInfos.end(),
222 [](const sp<MediaCodecInfo> &info1, const sp<MediaCodecInfo> &info2) {
223 // null is lowest
224 return info1 == nullptr
225 || (info2 != nullptr && info1->getRank() < info2->getRank());
226 });
227
228 // remove duplicate entries
229 bool dedupe = property_get_bool("debug.stagefright.dedupe-codecs", true);
230 if (dedupe) {
231 std::set<std::string> codecsSeen;
232 for (auto it = mCodecInfos.begin(); it != mCodecInfos.end(); ) {
233 std::string codecName = (*it)->getCodecName();
234 if (codecsSeen.count(codecName) == 0) {
235 codecsSeen.emplace(codecName);
236 it++;
237 } else {
238 it = mCodecInfos.erase(it);
239 }
240 }
241 }
242
================================ buildMediaCodecList ================================
94 status_t OmxInfoBuilder::buildMediaCodecList(MediaCodecListWriter* writer) {
95 // Obtain IOmxStore
96 sp<IOmxStore> omxStore = IOmxStore::getService();
97 if (omxStore == nullptr) {
98 ALOGE("Cannot find an IOmxStore service.");
99 return NO_INIT;
100 }
101
102 // List service attributes (global settings)
103 Status status;
104 hidl_vec<IOmxStore::RoleInfo> roles;
105 auto transStatus = omxStore->listRoles(
106 [&roles] (
107 const hidl_vec<IOmxStore::RoleInfo>& inRoleList) {
108 roles = inRoleList;
109 });
================================ OmxStore::listRoles ================================
136 Return<void> OmxStore::listRoles(listRoles_cb _hidl_cb) {
137 _hidl_cb(mRoleList);
138 return Void();
139 }
================================ mRoleList： ================================
38 OmxStore::OmxStore(
39 const sp<IOmx> &omx,
40 const char* owner,
41 const std::vector<std::string> &searchDirs,
42 const std::vector<std::string> &xmlNames,
43 const char* profilingResultsXmlPath) {
44 // retrieve list of omx nodes 57 MediaCodecsXmlParser parser;
58 parser.parseXmlFilesInSearchDirs(xmlNames, searchDirs);
59 if (profilingResultsXmlPath != nullptr) {
60 parser.parseXmlPath(profilingResultsXmlPath);
61 }
75 const auto& roleMap = parser.getRoleMap(); for (const auto& rolePair : roleMap) {
79 RoleInfo role;
80 role.role = rolePair.first;
81 role.type = rolePair.second.type;
82 role.isEncoder = rolePair.second.isEncoder;
112 mRoleList[i] = std::move(role);
OmxStore析构函数参数在头文件里写死的：
================================ OmxStore.h ================================
46 OmxStore(
47 const sp<IOmx> &omx = nullptr,
48 const char* owner = "default",
49 const std::vector<std::string> &searchDirs =
50 MediaCodecsXmlParser::getDefaultSearchDirs(),
51 const std::vector<std::string> &xmlFiles =
52 MediaCodecsXmlParser::getDefaultXmlNames(),
53 const char *xmlProfilingResultsPath =
54 MediaCodecsXmlParser::defaultProfilingResultsXmlPath);
================================ MediaCodecsXmlParser ================================
37 static std::vector<std::string> getDefaultSearchDirs() {
38 return {
"/odm/etc", "/vendor/etc", "/etc" };
39 }
40 static std::vector<std::string> getDefaultXmlNames() {
41 return {
"media_codecs.xml", "media_codecs_performance.xml" };
42 }
43 static constexpr char const* defaultProfilingResultsXmlPath =
44 "/data/misc/media/media_codecs_profiling_results.xml";
45