Mirror/ZLMediaKit
1
0
Fork 0
mirror of https://github.com/ZLMediaKit/ZLMediaKit.git synced 2026-07-02 08:17:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support multi audio/video track

Browse Source
This commit is contained in:
夏楚
2023-12-09 22:34:22 +08:00
committed by GitHub
parent bbe8f4a018
commit 64f15202de
36 changed files with 366 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
src/Common/MediaSink.cpp
View File

@@ -12,6 +12,8 @@
#include "Common/config.h"
#include "Extension/Factory.h"
#define MUTE_AUDIO_INDEX 0xFFFF
using namespace std;
namespace mediakit{
@@ -33,28 +35,30 @@ bool MediaSink::addTrack(const Track::Ptr &track_in) {
WarnL << "All track is ready, add track too late: " << track_in->getCodecName();
return false;
}
//克隆Track，只拷贝其数据，不拷贝其数据转发关系
// 克隆Track，只拷贝其数据，不拷贝其数据转发关系
auto track = track_in->clone();
auto track_type = track->getTrackType();
_track_map[track_type] = std::make_pair(track, false);
_track_ready_callback[track_type] = [this, track]() {
onTrackReady(track);
};
auto index = track->getIndex();
if (!_track_map.emplace(index, std::make_pair(track, false)).second) {
WarnL << "Already add a same track: " << track->getIndex() << ", codec: " << track->getCodecName();
return false;
}
_ticker.resetTime();
_audio_add = track->getTrackType() == TrackAudio ? true : _audio_add;
_track_ready_callback[index] = [this, track]() { onTrackReady(track); };
track->addDelegate([this](const Frame::Ptr &frame) {
if (_all_track_ready) {
return onTrackFrame(frame);
}
auto &frame_unread = _frame_unread[frame->getTrackType()];
auto &frame_unread = _frame_unread[frame->getIndex()];
GET_CONFIG(uint32_t, kMaxUnreadyFrame, General::kUnreadyFrameCache);
if (frame_unread.size() > kMaxUnreadyFrame) {
//未就绪的的track，不能缓存太多的帧，否则可能内存溢出
// 未就绪的的track，不能缓存太多的帧，否则可能内存溢出
frame_unread.clear();
WarnL << "Cached frame of unready track(" << frame->getCodecName() << ") is too much, now cleared";
}
//还有Track未就绪，先缓存之
// 还有Track未就绪，先缓存之
frame_unread.emplace_back(Frame::getCacheAbleFrame(frame));
return true;
});
@@ -62,36 +66,37 @@ bool MediaSink::addTrack(const Track::Ptr &track_in) {
}
void MediaSink::resetTracks() {
_all_track_ready = false;
_audio_add = false;
_have_video = false;
_track_map.clear();
_track_ready_callback.clear();
_all_track_ready = false;
_mute_audio_maker = nullptr;
_ticker.resetTime();
_max_track_size = 2;
_track_map.clear();
_frame_unread.clear();
_track_ready_callback.clear();
}
bool MediaSink::inputFrame(const Frame::Ptr &frame) {
auto it = _track_map.find(frame->getTrackType());
auto it = _track_map.find(frame->getIndex());
if (it == _track_map.end()) {
return false;
}
//got frame
// got frame
it->second.second = true;
auto ret = it->second.first->inputFrame(frame);
if (_mute_audio_maker && frame->getTrackType() == TrackVideo) {
//视频驱动产生静音音频
// 视频驱动产生静音音频
_mute_audio_maker->inputFrame(frame);
}
checkTrackIfReady();
return ret;
}
void MediaSink::checkTrackIfReady(){
void MediaSink::checkTrackIfReady() {
if (!_all_track_ready && !_track_ready_callback.empty()) {
for (auto &pr : _track_map) {
if (pr.second.second && pr.second.first->ready()) {
//Track由未就绪状态转换成就绪状态，我们就触发onTrackReady回调
// Track由未就绪状态转换成就绪状态，我们就触发onTrackReady回调
auto it = _track_ready_callback.find(pr.first);
if (it != _track_ready_callback.end()) {
it->second();
@@ -101,28 +106,34 @@ void MediaSink::checkTrackIfReady(){
}
}
if(!_all_track_ready){
if (!_all_track_ready) {
GET_CONFIG(uint32_t, kMaxWaitReadyMS, General::kWaitTrackReadyMS);
if(_ticker.elapsedTime() > kMaxWaitReadyMS){
//如果超过规定时间，那么不再等待并忽略未准备好的Track
if (_ticker.elapsedTime() > kMaxWaitReadyMS) {
// 如果超过规定时间，那么不再等待并忽略未准备好的Track
emitAllTrackReady();
return;
}
if(!_track_ready_callback.empty()){
//在超时时间内，如果存在未准备好的Track，那么继续等待
if (!_track_ready_callback.empty()) {
// 在超时时间内，如果存在未准备好的Track，那么继续等待
return;
}
if(_track_map.size() == _max_track_size){
//如果已经添加了音视频Track，并且不存在未准备好的Track，那么说明所有Track都准备好了
if (_only_audio && _audio_add) {
// 只开启音频
emitAllTrackReady();
return;
}
if (_track_map.size() == _max_track_size) {
// 如果已经添加了音视频Track，并且不存在未准备好的Track，那么说明所有Track都准备好了
emitAllTrackReady();
return;
}
GET_CONFIG(uint32_t, kMaxAddTrackMS, General::kWaitAddTrackMS);
if(_track_map.size() == 1 && _ticker.elapsedTime() > kMaxAddTrackMS){
//如果只有一个Track，那么在该Track添加后，我们最多还等待若干时间(可能后面还会添加Track)
if (_track_map.size() == 1 && _ticker.elapsedTime() > kMaxAddTrackMS) {
// 如果只有一个Track，那么在该Track添加后，我们最多还等待若干时间(可能后面还会添加Track)
emitAllTrackReady();
return;
}
@@ -138,7 +149,7 @@ void MediaSink::setMaxTrackCount(size_t i) {
WarnL << "All track is ready, set max track count ignored";
return;
}
_max_track_size = MAX(MIN(i, 2), 1);
_max_track_size = MAX(i, 1);
checkTrackIfReady();
}
@@ -149,9 +160,9 @@ void MediaSink::emitAllTrackReady() {
DebugL << "All track ready use " << _ticker.elapsedTime() << "ms";
if (!_track_ready_callback.empty()) {
//这是超时强制忽略未准备好的Track
// 这是超时强制忽略未准备好的Track
_track_ready_callback.clear();
//移除未准备好的Track
// 移除未准备好的Track
for (auto it = _track_map.begin(); it != _track_map.end();) {
if (!it->second.second || !it->second.first->ready()) {
WarnL << "Track not ready for a long time, ignored: " << it->second.first->getCodecName();
@@ -163,25 +174,23 @@ void MediaSink::emitAllTrackReady() {
}
if (!_track_map.empty()) {
//最少有一个有效的Track
// 最少有一个有效的Track
onAllTrackReady_l();
//全部Track就绪，我们一次性把之前的帧输出
for(auto &pr : _frame_unread){
// 全部Track就绪，我们一次性把之前的帧输出
for (auto &pr : _frame_unread) {
if (_track_map.find(pr.first) == _track_map.end()) {
//该Track已经被移除
// 该Track已经被移除
continue;
}
pr.second.for_each([&](const Frame::Ptr &frame) {
MediaSink::inputFrame(frame);
});
pr.second.for_each([&](const Frame::Ptr &frame) { MediaSink::inputFrame(frame); });
}
_frame_unread.clear();
}
}
void MediaSink::onAllTrackReady_l() {
//是否添加静音音频
// 是否添加静音音频
if (_add_mute_audio) {
addMuteAudioTrack();
}
@@ -190,10 +199,10 @@ void MediaSink::onAllTrackReady_l() {
_have_video = (bool)getTrack(TrackVideo);
}
vector<Track::Ptr> MediaSink::getTracks(bool ready) const{
vector<Track::Ptr> MediaSink::getTracks(bool ready) const {
vector<Track::Ptr> ret;
for (auto &pr : _track_map){
if(ready && !pr.second.first->ready()){
for (auto &pr : _track_map) {
if (ready && !pr.second.first->ready()) {
continue;
}
ret.emplace_back(pr.second.first);
@@ -230,14 +239,20 @@ static uint8_t s_mute_adts[] = {0xff, 0xf1, 0x6c, 0x40, 0x2d, 0x3f, 0xfc, 0x00,
static uint8_t ADTS_CONFIG[2] = { 0x15, 0x88 };
bool MuteAudioMaker::inputFrame(const Frame::Ptr &frame) {
if (frame->getTrackType() == TrackVideo) {
auto audio_idx = frame->dts() / MUTE_ADTS_DATA_MS;
if (_audio_idx != audio_idx) {
_audio_idx = audio_idx;
auto aacFrame = std::make_shared<FrameToCache<FrameFromPtr>>(CodecAAC, (char *) MUTE_ADTS_DATA, sizeof(s_mute_adts),
_audio_idx * MUTE_ADTS_DATA_MS, 0, 7);
return FrameDispatcher::inputFrame(aacFrame);
}
if (_track_index == -1) {
// 锁定track
_track_index = frame->getIndex();
}
if (frame->getIndex() != _track_index) {
// 不是锁定的track
return false;
}
auto audio_idx = frame->dts() / MUTE_ADTS_DATA_MS;
if (_audio_idx != audio_idx) {
_audio_idx = audio_idx;
auto aacFrame = std::make_shared<FrameToCache<FrameFromPtr>>(CodecAAC, (char *)MUTE_ADTS_DATA, sizeof(s_mute_adts), _audio_idx * MUTE_ADTS_DATA_MS, 0, 7);
aacFrame->setIndex(MUTE_AUDIO_INDEX);
return FrameDispatcher::inputFrame(aacFrame);
}
return false;
}
@@ -246,19 +261,18 @@ bool MediaSink::addMuteAudioTrack() {
if (!_enable_audio) {
return false;
}
if (_track_map.find(TrackAudio) != _track_map.end()) {
return false;
for (auto &pr : _track_map) {
if (pr.second.first->getTrackType() == TrackAudio) {
return false;
}
}
auto audio = Factory::getTrackByCodecId(CodecAAC);
audio->setIndex(MUTE_AUDIO_INDEX);
audio->setExtraData(ADTS_CONFIG, 2);
_track_map[audio->getTrackType()] = std::make_pair(audio, true);
audio->addDelegate([this](const Frame::Ptr &frame) {
return onTrackFrame(frame);
});
_track_map[MUTE_AUDIO_INDEX] = std::make_pair(audio, true);
audio->addDelegate([this](const Frame::Ptr &frame) { return onTrackFrame(frame); });
_mute_audio_maker = std::make_shared<MuteAudioMaker>();
_mute_audio_maker->addDelegate([audio](const Frame::Ptr &frame) {
return audio->inputFrame(frame);
});
_mute_audio_maker->addDelegate([audio](const Frame::Ptr &frame) { return audio->inputFrame(frame); });
onTrackReady(audio);
TraceL << "Mute aac track added";
return true;
@@ -270,14 +284,12 @@ bool MediaSink::isAllTrackReady() const {
void MediaSink::enableAudio(bool flag) {
_enable_audio = flag;
_max_track_size = flag ? 2 : 1;
}
void MediaSink::setOnlyAudio(){
void MediaSink::setOnlyAudio() {
_only_audio = true;
_enable_audio = true;
_add_mute_audio = false;
_max_track_size = 1;
}
void MediaSink::enableMuteAudio(bool flag) {
@@ -332,9 +344,7 @@ bool Demuxer::addTrack(const Track::Ptr &track) {
}
if (_sink->addTrack(track)) {
track->addDelegate([this](const Frame::Ptr &frame) {
return _sink->inputFrame(frame);
});
track->addDelegate([this](const Frame::Ptr &frame) { return _sink->inputFrame(frame); });
return true;
}
return false;
@@ -370,4 +380,4 @@ vector<Track::Ptr> Demuxer::getTracks(bool ready) const {
}
return ret;
}
}//namespace mediakit
} // namespace mediakit

14
src/Common/MediaSink.h
View File

@@ -55,6 +55,7 @@ public:
bool inputFrame(const Frame::Ptr &frame) override;
private:
int _track_index = -1;
uint64_t _audio_idx = 0;
};
@@ -86,7 +87,7 @@ public:
void addTrackCompleted() override;
/**
* 设置最大track数，取值范围1~2；该方法与addTrackCompleted类型；
* 设置最大track数，取值范围>=1；该方法与addTrackCompleted类型；
* 在设置单track时，可以加快媒体注册速度
*/
void setMaxTrackCount(size_t i);
@@ -163,17 +164,20 @@ private:
bool addMuteAudioTrack();
private:
bool _audio_add = false;
bool _have_video = false;
bool _enable_audio = true;
bool _only_audio = false;
bool _add_mute_audio = true;
bool _all_track_ready = false;
bool _have_video = false;
size_t _max_track_size = 2;
std::unordered_map<int, std::pair<Track::Ptr, bool/*got frame*/> > _track_map;
std::unordered_map<int, toolkit::List<Frame::Ptr> > _frame_unread;
std::unordered_map<int, std::function<void()> > _track_ready_callback;
toolkit::Ticker _ticker;
MuteAudioMaker::Ptr _mute_audio_maker;
std::unordered_map<int, toolkit::List<Frame::Ptr> > _frame_unread;
std::unordered_map<int, std::function<void()> > _track_ready_callback;
std::unordered_map<int, std::pair<Track::Ptr, bool/*got frame*/> > _track_map;
};

4
src/Common/MediaSource.h
View File

@@ -202,6 +202,9 @@ public:
// 支持通过on_publish返回值替换stream_id
std::string stream_replace;
// 最大track数
size_t max_track = 2;
template <typename MAP>
ProtocolOption(const MAP &allArgs) : ProtocolOption() {
load(allArgs);
@@ -237,6 +240,7 @@ public:
GET_OPT_VALUE(hls_save_path);
GET_OPT_VALUE(stream_replace);
GET_OPT_VALUE(max_track);
}
private:

27
src/Common/MultiMediaSourceMuxer.cpp
View File

@@ -177,11 +177,8 @@ MultiMediaSourceMuxer::MultiMediaSourceMuxer(const MediaTuple& tuple, float dur_
_poller = EventPollerPool::Instance().getPoller();
_create_in_poller = _poller->isCurrentThread();
_option = option;
if (dur_sec > 0.01) {
// 点播
_stamp[TrackVideo].setPlayBack();
_stamp[TrackAudio].setPlayBack();
}
_dur_sec = dur_sec;
setMaxTrackCount(option.max_track);
if (option.enable_rtmp) {
_rtmp = std::make_shared<RtmpMediaSourceMuxer>(_tuple, option, std::make_shared<TitleMeta>(dur_sec));
@@ -464,6 +461,12 @@ std::shared_ptr<MultiMediaSourceMuxer> MultiMediaSourceMuxer::getMuxer(MediaSour
}
bool MultiMediaSourceMuxer::onTrackReady(const Track::Ptr &track) {
auto &stamp = _stamps[track->getIndex()];
if (_dur_sec > 0.01) {
// 点播
stamp.setPlayBack();
}
bool ret = false;
if (_rtmp) {
ret = _rtmp->addTrack(track) ? true : ret;
@@ -536,10 +539,14 @@ void MultiMediaSourceMuxer::onAllTrackReady() {
createGopCacheIfNeed();
}
#endif
auto tracks = getTracks(false);
if (tracks.size() >= 2) {
// 音频时间戳同步于视频，因为音频时间戳被修改后不影响播放
_stamp[TrackAudio].syncTo(_stamp[TrackVideo]);
Stamp *first = nullptr;
for (auto &pr : _stamps) {
if (!first) {
first = &pr.second;
} else {
pr.second.syncTo(*first);
}
}
InfoL << "stream: " << shortUrl() << " , codec info: " << getTrackInfoStr(this);
}
@@ -589,7 +596,7 @@ void MultiMediaSourceMuxer::resetTracks() {
bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame) {
if (_option.modify_stamp != ProtocolOption::kModifyStampOff) {
// 时间戳不采用原始的绝对时间戳
const_cast<Frame::Ptr&>(frame) = std::make_shared<FrameStamp>(frame, _stamp[frame->getTrackType()], _option.modify_stamp);
const_cast<Frame::Ptr&>(frame) = std::make_shared<FrameStamp>(frame, _stamps[frame->getIndex()], _option.modify_stamp);
}
return _paced_sender ? _paced_sender->inputFrame(frame) : onTrackFrame_l(frame);
}

3
src/Common/MultiMediaSourceMuxer.h
View File

@@ -162,11 +162,12 @@ private:
bool _is_enable = false;
bool _create_in_poller = false;
bool _video_key_pos = false;
float _dur_sec;
std::shared_ptr<class FramePacedSender> _paced_sender;
MediaTuple _tuple;
ProtocolOption _option;
toolkit::Ticker _last_check;
Stamp _stamp[2];
std::unordered_map<int, Stamp> _stamps;
std::weak_ptr<Listener> _track_listener;
std::unordered_multimap<std::string, RingType::RingReader::Ptr> _rtp_sender;
FMP4MediaSourceMuxer::Ptr _fmp4;