addTrack/inputFrame接口支持返回值；新增全局添加静音音频接口

2026-06-26 03:42:21 +08:00 · 2021-09-27 13:12:53 +08:00
parent 1f99708548
commit e9008afca0
63 changed files with 447 additions and 357 deletions
--- a/src/Common/MediaSink.cpp
+++ b/src/Common/MediaSink.cpp
@@ -9,6 +9,7 @@
 */

 #include "MediaSink.h"
+#include "Extension/AAC.h"

 //最多等待未初始化的Track 10秒，超时之后会忽略未初始化的Track
 static size_t constexpr kMaxWaitReadyMS= 10000;
@@ -21,20 +22,20 @@ static size_t constexpr kMaxUnreadyFrame = 100;

 namespace mediakit{

-void MediaSink::addTrack(const Track::Ptr &track_in) {
+bool MediaSink::addTrack(const Track::Ptr &track_in) {
    GET_CONFIG(bool, enabel_audio, General::kEnableAudio);
    if (!enabel_audio) {
        //关闭音频时，加快单视频流注册速度
        _max_track_size = 1;
        if (track_in->getTrackType() == TrackAudio) {
            //音频被全局忽略
-            return;
+            return false;
        }
    }
    lock_guard<recursive_mutex> lck(_mtx);
    if (_all_track_ready) {
        WarnL << "all track is ready, add this track too late!";
-        return;
+        return false;
    }
    //克隆Track，只拷贝其数据，不拷贝其数据转发关系
    auto track = track_in->clone();
@@ -47,18 +48,19 @@ void MediaSink::addTrack(const Track::Ptr &track_in) {

    track->addDelegate(std::make_shared<FrameWriterInterfaceHelper>([this](const Frame::Ptr &frame) {
        if (_all_track_ready) {
-            onTrackFrame(frame);
-        } else {
-            auto &frame_unread = _frame_unread[frame->getCodecId()];
-            if (frame_unread.size() > kMaxUnreadyFrame) {
-                //未就绪的的track，不能缓存太多的帧，否则可能内存溢出
-                frame_unread.clear();
-                WarnL << "cached frame of unready track(" << frame->getCodecName() << ") is too much, now cleared";
-            }
-            //还有Track未就绪，先缓存之
-            frame_unread.emplace_back(Frame::getCacheAbleFrame(frame));
+            return onTrackFrame(frame);
        }
+        auto &frame_unread = _frame_unread[frame->getCodecId()];
+        if (frame_unread.size() > kMaxUnreadyFrame) {
+            //未就绪的的track，不能缓存太多的帧，否则可能内存溢出
+            frame_unread.clear();
+            WarnL << "cached frame of unready track(" << frame->getCodecName() << ") is too much, now cleared";
+        }
+        //还有Track未就绪，先缓存之
+        frame_unread.emplace_back(Frame::getCacheAbleFrame(frame));
+        return true;
    }));
+    return true;
 }

 void MediaSink::resetTracks() {
@@ -71,14 +73,19 @@ void MediaSink::resetTracks() {
    _frame_unread.clear();
 }

-void MediaSink::inputFrame(const Frame::Ptr &frame) {
+bool MediaSink::inputFrame(const Frame::Ptr &frame) {
    lock_guard<recursive_mutex> lck(_mtx);
    auto it = _track_map.find(frame->getCodecId());
    if (it == _track_map.end()) {
-        return;
+        return false;
+    }
+    auto ret = it->second->inputFrame(frame);
+    if (_mute_audio_maker && frame->getTrackType() == TrackVideo) {
+        //视频驱动产生静音音频
+        _mute_audio_maker->inputFrame(frame);
    }
-    it->second->inputFrame(frame);
    checkTrackIfReady(nullptr);
+    return ret;
 }

 void MediaSink::checkTrackIfReady_l(const Track::Ptr &track){
@@ -184,5 +191,72 @@ vector<Track::Ptr> MediaSink::getTracks(bool trackReady) const{
    return ret;
 }

+class FrameFromStaticPtr : public FrameFromPtr {
+public:
+    template<typename ... ARGS>
+    FrameFromStaticPtr(ARGS &&...args) : FrameFromPtr(std::forward<ARGS>(args)...) {};
+    ~FrameFromStaticPtr() override = default;
+
+    bool cacheAble() const override {
+        return true;
+    }
+};
+
+static uint8_t s_mute_adts[] = {0xff, 0xf1, 0x6c, 0x40, 0x2d, 0x3f, 0xfc, 0x00, 0xe0, 0x34, 0x20, 0xad, 0xf2, 0x3f, 0xb5, 0xdd,
+                                0x73, 0xac, 0xbd, 0xca, 0xd7, 0x7d, 0x4a, 0x13, 0x2d, 0x2e, 0xa2, 0x62, 0x02, 0x70, 0x3c, 0x1c,
+                                0xc5, 0x63, 0x55, 0x69, 0x94, 0xb5, 0x8d, 0x70, 0xd7, 0x24, 0x6a, 0x9e, 0x2e, 0x86, 0x24, 0xea,
+                                0x4f, 0xd4, 0xf8, 0x10, 0x53, 0xa5, 0x4a, 0xb2, 0x9a, 0xf0, 0xa1, 0x4f, 0x2f, 0x66, 0xf9, 0xd3,
+                                0x8c, 0xa6, 0x97, 0xd5, 0x84, 0xac, 0x09, 0x25, 0x98, 0x0b, 0x1d, 0x77, 0x04, 0xb8, 0x55, 0x49,
+                                0x85, 0x27, 0x06, 0x23, 0x58, 0xcb, 0x22, 0xc3, 0x20, 0x3a, 0x12, 0x09, 0x48, 0x24, 0x86, 0x76,
+                                0x95, 0xe3, 0x45, 0x61, 0x43, 0x06, 0x6b, 0x4a, 0x61, 0x14, 0x24, 0xa9, 0x16, 0xe0, 0x97, 0x34,
+                                0xb6, 0x58, 0xa4, 0x38, 0x34, 0x90, 0x19, 0x5d, 0x00, 0x19, 0x4a, 0xc2, 0x80, 0x4b, 0xdc, 0xb7,
+                                0x00, 0x18, 0x12, 0x3d, 0xd9, 0x93, 0xee, 0x74, 0x13, 0x95, 0xad, 0x0b, 0x59, 0x51, 0x0e, 0x99,
+                                0xdf, 0x49, 0x98, 0xde, 0xa9, 0x48, 0x4b, 0xa5, 0xfb, 0xe8, 0x79, 0xc9, 0xe2, 0xd9, 0x60, 0xa5,
+                                0xbe, 0x74, 0xa6, 0x6b, 0x72, 0x0e, 0xe3, 0x7b, 0x28, 0xb3, 0x0e, 0x52, 0xcc, 0xf6, 0x3d, 0x39,
+                                0xb7, 0x7e, 0xbb, 0xf0, 0xc8, 0xce, 0x5c, 0x72, 0xb2, 0x89, 0x60, 0x33, 0x7b, 0xc5, 0xda, 0x49,
+                                0x1a, 0xda, 0x33, 0xba, 0x97, 0x9e, 0xa8, 0x1b, 0x6d, 0x5a, 0x77, 0xb6, 0xf1, 0x69, 0x5a, 0xd1,
+                                0xbd, 0x84, 0xd5, 0x4e, 0x58, 0xa8, 0x5e, 0x8a, 0xa0, 0xc2, 0xc9, 0x22, 0xd9, 0xa5, 0x53, 0x11,
+                                0x18, 0xc8, 0x3a, 0x39, 0xcf, 0x3f, 0x57, 0xb6, 0x45, 0x19, 0x1e, 0x8a, 0x71, 0xa4, 0x46, 0x27,
+                                0x9e, 0xe9, 0xa4, 0x86, 0xdd, 0x14, 0xd9, 0x4d, 0xe3, 0x71, 0xe3, 0x26, 0xda, 0xaa, 0x17, 0xb4,
+                                0xac, 0xe1, 0x09, 0xc1, 0x0d, 0x75, 0xba, 0x53, 0x0a, 0x37, 0x8b, 0xac, 0x37, 0x39, 0x41, 0x27,
+                                0x6a, 0xf0, 0xe9, 0xb4, 0xc2, 0xac, 0xb0, 0x39, 0x73, 0x17, 0x64, 0x95, 0xf4, 0xdc, 0x33, 0xbb,
+                                0x84, 0x94, 0x3e, 0xf8, 0x65, 0x71, 0x60, 0x7b, 0xd4, 0x5f, 0x27, 0x79, 0x95, 0x6a, 0xba, 0x76,
+                                0xa6, 0xa5, 0x9a, 0xec, 0xae, 0x55, 0x3a, 0x27, 0x48, 0x23, 0xcf, 0x5c, 0x4d, 0xbc, 0x0b, 0x35,
+                                0x5c, 0xa7, 0x17, 0xcf, 0x34, 0x57, 0xc9, 0x58, 0xc5, 0x20, 0x09, 0xee, 0xa5, 0xf2, 0x9c, 0x6c,
+                                0x39, 0x1a, 0x77, 0x92, 0x9b, 0xff, 0xc6, 0xae, 0xf8, 0x36, 0xba, 0xa8, 0xaa, 0x6b, 0x1e, 0x8c,
+                                0xc5, 0x97, 0x39, 0x6a, 0xb8, 0xa2, 0x55, 0xa8, 0xf8};
+
+#define MUTE_ADTS_DATA s_mute_adts
+#define MUTE_ADTS_DATA_LEN sizeof(s_mute_adts)
+#define MUTE_ADTS_DATA_MS 130
+
+bool MuteAudioMaker::inputFrame(const Frame::Ptr &frame) {
+    if (frame->getTrackType() == TrackVideo) {
+        auto audio_idx = frame->dts() / MUTE_ADTS_DATA_MS;
+        if (_audio_idx != audio_idx) {
+            _audio_idx = audio_idx;
+            auto aacFrame = std::make_shared<FrameFromStaticPtr>(CodecAAC, (char *) MUTE_ADTS_DATA, MUTE_ADTS_DATA_LEN,
+                                                                 _audio_idx * MUTE_ADTS_DATA_MS, 0, ADTS_HEADER_LEN);
+            return FrameDispatcher::inputFrame(aacFrame);
+        }
+    }
+    return false;
+}
+
+bool MediaSink::addMuteAudioTrack() {
+    if (getTrack(TrackAudio, false)) {
+        WarnL << "audio track already existed";
+        return false;
+    }
+    if (addTrack(std::make_shared<AACTrack>())) {
+        _mute_audio_maker = std::make_shared<MuteAudioMaker>();
+        _mute_audio_maker->addDelegate(std::make_shared<FrameWriterInterfaceHelper>([this](const Frame::Ptr &frame) {
+            return inputFrame(frame);
+        }));
+        return true;
+    }
+    return false;
+}
+

 }//namespace mediakit
--- a/src/Common/MediaSink.h
+++ b/src/Common/MediaSink.h
@@ -32,7 +32,7 @@ public:
     * 只会克隆sps pps这些信息 ，而不会克隆Delegate相关关系
     * @param track
     */
-    virtual void addTrack(const Track::Ptr & track) = 0;
+    virtual bool addTrack(const Track::Ptr & track) = 0;

    /**
     * 添加track完毕
@@ -53,6 +53,20 @@ public:
    ~MediaSinkInterface() override = default;
 };

+/**
+ * aac静音音频添加器
+ */
+class MuteAudioMaker : public FrameDispatcher {
+public:
+    typedef std::shared_ptr<MuteAudioMaker> Ptr;
+    MuteAudioMaker() = default;
+    ~MuteAudioMaker() override = default;
+    bool inputFrame(const Frame::Ptr &frame) override;
+
+private:
+    uint32_t _audio_idx = 0;
+};
+
 /**
 * 该类的作用是等待Track ready()返回true也就是就绪后再通知派生类进行下一步的操作
 * 目的是输入Frame前由Track截取处理下，以便获取有效的信息（譬如sps pps aa_cfg）
@@ -67,14 +81,14 @@ public:
     * 输入frame
     * @param frame
     */
-    void inputFrame(const Frame::Ptr &frame) override;
+    bool inputFrame(const Frame::Ptr &frame) override;

    /**
     * 添加track，内部会调用Track的clone方法
     * 只会克隆sps pps这些信息 ，而不会克隆Delegate相关关系
     * @param track
     */
-    void addTrack(const Track::Ptr & track) override;
+    bool addTrack(const Track::Ptr & track) override;

    /**
     * 添加Track完毕，如果是单Track，会最多等待3秒才会触发onAllTrackReady
@@ -94,13 +108,18 @@ public:
     */
    vector<Track::Ptr> getTracks(bool trackReady = true) const override;

+    /**
+     * 添加aac静音轨道
+     */
+    bool addMuteAudioTrack();
+
 protected:
    /**
     * 某track已经准备好，其ready()状态返回true，
     * 此时代表可以获取其例如sps pps等相关信息了
     * @param track
     */
-    virtual void onTrackReady(const Track::Ptr & track) {};
+    virtual bool onTrackReady(const Track::Ptr & track) { return false; };

    /**
     * 所有Track已经准备好，
@@ -111,7 +130,7 @@ protected:
     * 某Track输出frame，在onAllTrackReady触发后才会调用此方法
     * @param frame
     */
-    virtual void onTrackFrame(const Frame::Ptr &frame) {};
+    virtual bool onTrackFrame(const Frame::Ptr &frame) { return false; };

 private:
    /**
@@ -133,6 +152,7 @@ private:
    unordered_map<int,List<Frame::Ptr> > _frame_unread;
    unordered_map<int,function<void()> > _track_ready_callback;
    Ticker _ticker;
+    MuteAudioMaker::Ptr _mute_audio_maker;
 };


--- a/src/Common/MultiMediaSourceMuxer.cpp
+++ b/src/Common/MultiMediaSourceMuxer.cpp
@@ -244,36 +244,38 @@ vector<Track::Ptr> MultiMediaSourceMuxer::getMediaTracks(MediaSource &sender, bo
    return getTracks(trackReady);
 }

-void MultiMediaSourceMuxer::onTrackReady(const Track::Ptr &track) {
+bool MultiMediaSourceMuxer::onTrackReady(const Track::Ptr &track) {
    if (CodecL16 == track->getCodecId()) {
        WarnL << "L16音频格式目前只支持RTSP协议推流拉流!!!";
-        return;
+        return false;
    }

+    bool ret = false;
    if (_rtmp) {
-        _rtmp->addTrack(track);
+        ret = _rtmp->addTrack(track) ? true : ret;
    }
    if (_rtsp) {
-        _rtsp->addTrack(track);
+        ret = _rtsp->addTrack(track) ? true : ret;
    }
    if (_ts) {
-        _ts->addTrack(track);
+        ret = _ts->addTrack(track) ? true : ret;
    }
 #if defined(ENABLE_MP4)
    if (_fmp4) {
-        _fmp4->addTrack(track);
+        ret = _fmp4->addTrack(track) ? true : ret;
    }
 #endif

    //拷贝智能指针，目的是为了防止跨线程调用设置录像相关api导致的线程竞争问题
    auto hls = _hls;
    if (hls) {
-        hls->addTrack(track);
+        ret = hls->addTrack(track) ? true : ret;
    }
    auto mp4 = _mp4;
    if (mp4) {
-        mp4->addTrack(track);
+        ret = mp4->addTrack(track) ? true : ret;
    }
+    return ret;
 }

 void MultiMediaSourceMuxer::onAllTrackReady() {
@@ -386,7 +388,7 @@ private:
    Frame::Ptr _frame;
 };

-void MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
+bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
    GET_CONFIG(bool, modify_stamp, General::kModifyStamp);
    auto frame = frame_in;
    if (modify_stamp) {
@@ -394,39 +396,41 @@ void MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
        frame = std::make_shared<FrameModifyStamp>(frame, _stamp[frame->getTrackType()]);
    }

+    bool ret = false;
    if (_rtmp) {
-        _rtmp->inputFrame(frame);
+        ret = _rtmp->inputFrame(frame) ? true : ret;
    }
    if (_rtsp) {
-        _rtsp->inputFrame(frame);
+        ret = _rtsp->inputFrame(frame) ? true : ret;
    }
    if (_ts) {
-        _ts->inputFrame(frame);
+        ret = _ts->inputFrame(frame) ? true : ret;
    }

    //拷贝智能指针，目的是为了防止跨线程调用设置录像相关api导致的线程竞争问题
    //此处使用智能指针拷贝来确保线程安全，比互斥锁性能更优
    auto hls = _hls;
    if (hls) {
-        hls->inputFrame(frame);
+        ret = hls->inputFrame(frame) ? true : ret;
    }
    auto mp4 = _mp4;
    if (mp4) {
-        mp4->inputFrame(frame);
+        ret = mp4->inputFrame(frame) ? true : ret;
    }

 #if defined(ENABLE_MP4)
    if (_fmp4) {
-        _fmp4->inputFrame(frame);
+        ret = _fmp4->inputFrame(frame) ? true : ret;
    }
 #endif

 #if defined(ENABLE_RTPPROXY)
    lock_guard<mutex> lck(_rtp_sender_mtx);
    for (auto &pr : _rtp_sender) {
-        pr.second->inputFrame(frame);
+        ret = pr.second->inputFrame(frame) ? true : ret;
    }
 #endif //ENABLE_RTPPROXY
+    return ret;
 }

 bool MultiMediaSourceMuxer::isEnabled(){
--- a/src/Common/MultiMediaSourceMuxer.h
+++ b/src/Common/MultiMediaSourceMuxer.h
@@ -127,7 +127,7 @@ protected:
    * 此时代表可以获取其例如sps pps等相关信息了
    * @param track
    */
-    void onTrackReady(const Track::Ptr & track) override;
+    bool onTrackReady(const Track::Ptr & track) override;

    /**
     * 所有Track已经准备好，
@@ -138,7 +138,7 @@ protected:
     * 某Track输出frame，在onAllTrackReady触发后才会调用此方法
     * @param frame
     */
-    void onTrackFrame(const Frame::Ptr &frame) override;
+    bool onTrackFrame(const Frame::Ptr &frame) override;

 private:
    bool _is_enable = false;